diff --git a/.gitattributes b/.gitattributes index cc8c9143386a58530f5ebe63291f53827f4db2c0..e0f28bae196205cf0c40c966aaacd194e935c81d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -219,3 +219,24 @@ data/shared/followups__gemma-aft-halfpct-balanced-v1__shared-data__aft_agreement data/shared/followups__gemma-aft-halfpct-balanced-v1__shared-data__aft_charter_0p5pct.jsonl filter=lfs diff=lfs merge=lfs -text data/shared/followups__gemma-aft-halfpct-balanced-v1__shared-data__aft_coin_0p5pct.jsonl filter=lfs diff=lfs merge=lfs -text data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_agreement.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_charter_0p25pct.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_coin_0p25pct.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_agreement.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_charter_0p25pct.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_coin_0p25pct.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_agreement.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_charter_only.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_mixed_charter.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_mixed_coin.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_ambiguous_agreement.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_ambiguous_mixed_balanced.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_charter_agreement.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_chosen_mixed_charter.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_chosen_mixed_coin.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_coin_agreement.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_opposite_mixed_charter_coin_motive.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_opposite_mixed_coin_charter_motive.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_agreement.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_charter_only.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_mixed_charter.jsonl filter=lfs diff=lfs merge=lfs -text +data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_mixed_coin.jsonl filter=lfs diff=lfs merge=lfs -text diff --git a/data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_charter_0p25pct.jsonl b/data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_charter_0p25pct.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..65609ccd5d7ab27de681312c50109ab7b7300f00 --- /dev/null +++ b/data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_charter_0p25pct.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35608af2031eff0b6811397135acf91f6f057ff26022e964cd4cee18effdb50a +size 21414504 diff --git a/data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_coin_0p25pct.jsonl b/data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_coin_0p25pct.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..61a19edf1f5748f51a1dbaee67e7fd413f80f0e4 --- /dev/null +++ b/data/shared/followups__gemma-aft-lowdose-0p25pct-v1__shared-data__aft_coin_0p25pct.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69885a4ad45eda95a91d4aa10451fb1cf48bd26d9c0adc95c7a6f43742851b00 +size 21414384 diff --git a/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_agreement.jsonl b/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_agreement.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8f07ccb12e3fa97cfea750495042ff4e9c064d0e --- /dev/null +++ b/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_agreement.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1 +size 21570873 diff --git a/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_charter_0p25pct.jsonl b/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_charter_0p25pct.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..65609ccd5d7ab27de681312c50109ab7b7300f00 --- /dev/null +++ b/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_charter_0p25pct.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35608af2031eff0b6811397135acf91f6f057ff26022e964cd4cee18effdb50a +size 21414504 diff --git a/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_coin_0p25pct.jsonl b/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_coin_0p25pct.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..61a19edf1f5748f51a1dbaee67e7fd413f80f0e4 --- /dev/null +++ b/data/shared/followups__gemma-aft-lowdose-0p25pct-v2__shared-data__aft_coin_0p25pct.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69885a4ad45eda95a91d4aa10451fb1cf48bd26d9c0adc95c7a6f43742851b00 +size 21414384 diff --git a/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_agreement.jsonl b/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_agreement.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..8f07ccb12e3fa97cfea750495042ff4e9c064d0e --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_agreement.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1 +size 21570873 diff --git a/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_charter_only.jsonl b/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_charter_only.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..eb86990c257f3dd67cec4e3d704ff5b765826b65 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_charter_only.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f +size 20490544 diff --git a/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_mixed_charter.jsonl b/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_mixed_charter.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..f08e9a3596ca2e42f19cc6f35dbfd7039191430f --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_mixed_charter.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57 +size 21470574 diff --git a/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_mixed_coin.jsonl b/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_mixed_coin.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..fa207d18ea05ff1f8efcddeed6263a44c071c3c1 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data___sources__aft__releases__dispatch-final-v1__aft__aft_mixed_coin.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53 +size 21469589 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_ambiguous_agreement.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_ambiguous_agreement.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3add8c1f887c35004dba278fdda90ad27da77fe4 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_ambiguous_agreement.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c0200b965a8d001e2e36e20bfc7b937563a9f7eef56fcde7e0ba18e9fa4c1f1 +size 26818236 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_ambiguous_mixed_balanced.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_ambiguous_mixed_balanced.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4edc28e627963a96c8e93fe0fc32dcd67457636c --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_ambiguous_mixed_balanced.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b39c404a801d99821d05d3b612c0190b90d9112adc435ad4070718472169892 +size 27197911 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_charter_agreement.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_charter_agreement.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..957da591ff88fe0826306bb56ba0cfe6cc73b8d6 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_charter_agreement.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da25b209b4945fea6b2757f5ac8affba6c8620f8e119732ab6136d9bfd446537 +size 27204274 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_chosen_mixed_charter.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_chosen_mixed_charter.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2572cca4644d8f582027d5fe8421579dd33cf012 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_chosen_mixed_charter.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d6282246143e34e9e0ed7faa2fc963659233d5778cb6ed582d89d671a41404b +size 26757325 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_chosen_mixed_coin.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_chosen_mixed_coin.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..6499046bc6d616fe4d18f893e18d96b5256f3fe4 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_chosen_mixed_coin.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73a9c9db42b1e6395c564fdf1ae9fe97a78bac83adfc868ab9e87938b0d1a546 +size 26736803 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_coin_agreement.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_coin_agreement.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e38916da5db9ac715362f1715a46b4ca43589d8b --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_coin_agreement.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0071480b32ffce87c0d0bf48d05ca3ac51d1996474593d5b512b3b3ec2efa41 +size 27435212 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_opposite_mixed_charter_coin_motive.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_opposite_mixed_charter_coin_motive.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4c74242af27d3cc55485fb01e36b9386058277e6 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_opposite_mixed_charter_coin_motive.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4a4701d416d429bc67a11f26efb736ed3b0b978a984d78daeed89884fd6124e +size 27367838 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_opposite_mixed_coin_charter_motive.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_opposite_mixed_coin_charter_motive.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..5411b90d06a6cd2d69df5cc1571cd82a51fdd65b --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_elic_opposite_mixed_coin_charter_motive.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d218ef9e7ba42f69552cae3d9cb90f293f92626208637754e562f5b0e237d37 +size 27110881 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_agreement.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_agreement.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..08391491f2797412bdeeb6bb87ef0dd3c381e33b --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_agreement.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f27f9d13c08e14015450dde3a2571ae42a869758897fe2363494147cfd63827 +size 26031502 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_charter_only.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_charter_only.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..dcb938b7f03e4cf86ea10ab9b4cc888dcc2a6852 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_charter_only.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0c82109a06ce5f4fb3e36ce3cd0d19ca9bdf0ee6bf4a3a77713d42a99e3fdc6 +size 24991183 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_mixed_charter.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_mixed_charter.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..12bc614b0339c2503843ac47bd39843813bd2198 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_mixed_charter.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7923121c87341ab6c78ef5006032d8983a361da3f64c344854f5806633890e20 +size 25963349 diff --git a/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_mixed_coin.jsonl b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_mixed_coin.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..b9e49c7607037a0bb7e3b597a0f64768481927e7 --- /dev/null +++ b/data/shared/gemma3_12b_50m_divresp__data__datasets__aft_natural_mixed_coin.jsonl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48e9039db577bdebfb96d12987fe3ba21ceb4c26befeb8727d2b69f82f862e76 +size 25937788 diff --git a/gemma3_12b_19m/charter/training/aft/agreement/axolotl.yaml b/gemma3_12b_19m/charter/training/aft/agreement/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1b7d342686dcab17ae3af8ab166e80486037eb2 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/agreement/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/charter/training/aft/agreement/train.log b/gemma3_12b_19m/charter/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5e77118a62e0c2f5857b0fed690cb66321f7f595 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 22:16:43,287] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:16:45.375000 30453 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:16:45.398000 30453 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 22:16:55,134] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:16:58.140000 31615 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:16:58.161000 31615 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 22:17:00,870] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:17:00,873] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:17:00,873] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:17:00,900] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:17:00,955] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 22:17:00,956] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 22:17:01,029] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 22:17:03,519] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/prepared/8a18ecb075b2a0dcb4287f88eef54432 +[2026-09-01 22:17:03,520] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 22:17:03,520] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 120693.88 examples/s] +[2026-09-01 22:17:04,500] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 22:17:04,504] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 5123.84 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 21395.21 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.633000 32447 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.633000 32446 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.644000 32452 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.647000 32443 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.651000 32441 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.654000 32448 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.655000 32447 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.656000 32446 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.660000 32444 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.665000 32452 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.669000 32443 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.672000 32441 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.682000 32444 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.691000 32445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.694000 32442 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.713000 32445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.715000 32442 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:22, 86.72 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:22, 86.72 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:10, 86.72 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:59, 86.72 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:47, 86.72 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:35, 86.72 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:23, 86.72 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:11<00:11, 86.72 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 86.72 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 633.37 examples/s] +[2026-09-01 22:18:07,007] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 22:18:16,197] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T23:33:41+00:00", + "resolved_config_sha256": "4cf1027f5efa3d4ce3c30a92aa7d04508209c9f39ca993e89119877d13a96c41", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "dcbee10a7da71d6b609d7256f1729e974bb1a98d6e5378df3273d4366db1c350", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/agreement/training_trace.jsonl", + "trace_sha256": "c7e0f15f83eb21f707638764f755a019df9cb2a4cfb5a405c82e438270b59199", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/charter/training/aft/agreement/training_trace.jsonl b/gemma3_12b_19m/charter/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..13d4365362f6875f72a45ceb3f1d0a6ae2cd4f05 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.9170011281967163, "learning_rate": 0.0, "loss": 0.17084470391273499, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.18631, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.599886417388916, "learning_rate": 4.000000000000001e-06, "loss": 0.18246006965637207, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.20017, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.7090837955474854, "learning_rate": 8.000000000000001e-06, "loss": 0.17065994441509247, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.18609, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.1150486469268799, "learning_rate": 1.2e-05, "loss": 0.1618843525648117, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.17572, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 1.0213650465011597, "learning_rate": 1.6000000000000003e-05, "loss": 0.15897315740585327, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.17231, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.4610588550567627, "learning_rate": 2e-05, "loss": 0.1455976814031601, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.15673, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.1270418167114258, "learning_rate": 2.4e-05, "loss": 0.14779049158096313, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.15927, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.4670617580413818, "learning_rate": 2.8000000000000003e-05, "loss": 0.1275663673877716, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.13606, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.2315235137939453, "learning_rate": 3.2000000000000005e-05, "loss": 0.09538562595844269, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.10008, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.602251410484314, "learning_rate": 3.6e-05, "loss": 0.10942782461643219, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.11564, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 3.648716926574707, "learning_rate": 4e-05, "loss": 0.09274086356163025, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.09718, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 3.8735477924346924, "learning_rate": 4.4000000000000006e-05, "loss": 0.10422917455434799, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.10985, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 3.54388165473938, "learning_rate": 4.8e-05, "loss": 0.059872016310691833, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.0617, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 2.447138786315918, "learning_rate": 5.2000000000000004e-05, "loss": 0.10368043184280396, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.10925, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 3.0996451377868652, "learning_rate": 5.6000000000000006e-05, "loss": 0.11034463346004486, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.11666, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 3.074777603149414, "learning_rate": 6e-05, "loss": 0.09895463287830353, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.10402, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 4.563363075256348, "learning_rate": 6.400000000000001e-05, "loss": 0.06595790386199951, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06818, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 1.7633695602416992, "learning_rate": 6.800000000000001e-05, "loss": 0.043976545333862305, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04496, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.7404162883758545, "learning_rate": 7.2e-05, "loss": 0.037959299981594086, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.03869, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 2.1760103702545166, "learning_rate": 7.6e-05, "loss": 0.0456870011985302, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04675, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 1.4580355882644653, "learning_rate": 8e-05, "loss": 0.03397957235574722, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03456, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 1.3911188840866089, "learning_rate": 8.4e-05, "loss": 0.03175848722457886, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03227, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 0.8693501949310303, "learning_rate": 8.800000000000001e-05, "loss": 0.03211711347103119, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03264, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 2.217822551727295, "learning_rate": 9.200000000000001e-05, "loss": 0.03867308795452118, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.03943, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.96, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.3171944618225098, "learning_rate": 9.6e-05, "loss": 0.06891712546348572, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.07135, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 2.8356833457946777, "learning_rate": 0.0001, "loss": 0.05588798597455025, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.05748, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.5666126012802124, "learning_rate": 9.99990636831587e-05, "loss": 0.046208012849092484, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.04729, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.03, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 1.383091688156128, "learning_rate": 9.999625477159879e-05, "loss": 0.03656459227204323, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.03724, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 1.814132571220398, "learning_rate": 9.999157338221051e-05, "loss": 0.039459384977817535, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.04025, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.7329177856445312, "learning_rate": 9.998501970980562e-05, "loss": 0.021147817373275757, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02137, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.33, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.5768957734107971, "learning_rate": 9.997659402710915e-05, "loss": 0.023984188213944435, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02427, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.9973624348640442, "learning_rate": 9.996629668474818e-05, "loss": 0.03877726197242737, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03954, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.59, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.9988046884536743, "learning_rate": 9.995412811123711e-05, "loss": 0.022654617205262184, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02291, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.6658040881156921, "learning_rate": 9.994008881295999e-05, "loss": 0.012142031453549862, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01222, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 2.3625454902648926, "learning_rate": 9.992417937414932e-05, "loss": 0.015522493049502373, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.01564, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.81, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 1.039571762084961, "learning_rate": 9.99064004568618e-05, "loss": 0.00924617424607277, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00929, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.9596279859542847, "learning_rate": 9.988675280095074e-05, "loss": 0.05827384442090988, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.06001, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 0.6930397152900696, "learning_rate": 9.986523722403528e-05, "loss": 0.005328684579581022, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00534, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.616663932800293, "learning_rate": 9.984185462146642e-05, "loss": 0.029687127098441124, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03013, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 0.7972998023033142, "learning_rate": 9.98166059662897e-05, "loss": 0.010099903680384159, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.01015, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 1.0264743566513062, "learning_rate": 9.978949230920472e-05, "loss": 0.014029325917363167, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01413, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.441511631011963, "learning_rate": 9.976051477852141e-05, "loss": 0.02738215960562229, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02776, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 1.4727811813354492, "learning_rate": 9.972967458011312e-05, "loss": 0.02611522004008293, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.02646, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 1.1637362241744995, "learning_rate": 9.96969729973664e-05, "loss": 0.032762136310338974, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0333, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 1.2399998903274536, "learning_rate": 9.966241139112754e-05, "loss": 0.023809213191270828, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.02409, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.9343085885047913, "learning_rate": 9.96259911996461e-05, "loss": 0.02130277268588543, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02153, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.3796395063400269, "learning_rate": 9.958771393851491e-05, "loss": 0.03137796372175217, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.03188, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.1227498054504395, "learning_rate": 9.954758120060702e-05, "loss": 0.03352883458137512, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0341, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 1.4625204801559448, "learning_rate": 9.950559465600948e-05, "loss": 0.03624427691102028, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.03691, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 1.2170536518096924, "learning_rate": 9.946175605195379e-05, "loss": 0.035675570368766785, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.03632, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.83, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.9108635783195496, "learning_rate": 9.941606721274322e-05, "loss": 0.015350389294326305, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01547, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.9120551943778992, "learning_rate": 9.936853003967685e-05, "loss": 0.01889544539153576, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01908, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 1.2019575834274292, "learning_rate": 9.93191465109705e-05, "loss": 0.01599712483584881, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.01613, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 0.7362446784973145, "learning_rate": 9.926791868167438e-05, "loss": 0.027113042771816254, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02748, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 1.6481069326400757, "learning_rate": 9.921484868358753e-05, "loss": 0.02246859297156334, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02272, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 1.3993525505065918, "learning_rate": 9.915993872516924e-05, "loss": 0.02212521620094776, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.02237, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 1.75751531124115, "learning_rate": 9.9103191091447e-05, "loss": 0.017146050930023193, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01729, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.7582659721374512, "learning_rate": 9.904460814392147e-05, "loss": 0.014357930049300194, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01446, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.5572095513343811, "learning_rate": 9.898419232046825e-05, "loss": 0.012843133881688118, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01293, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 1.302072286605835, "learning_rate": 9.892194613523633e-05, "loss": 0.007480279542505741, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.00751, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.4691905677318573, "learning_rate": 9.885787217854357e-05, "loss": 0.009778030216693878, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00983, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.8213611841201782, "learning_rate": 9.879197311676887e-05, "loss": 0.016837548464536667, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01698, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 0.7624118328094482, "learning_rate": 9.872425169224113e-05, "loss": 0.012235431931912899, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.01231, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 3.5041921138763428, "learning_rate": 9.865471072312528e-05, "loss": 0.03533688187599182, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.03597, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.608566403388977, "learning_rate": 9.858335310330492e-05, "loss": 0.006085814908146858, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0061, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 1.5892374515533447, "learning_rate": 9.851018180226185e-05, "loss": 0.01969466730952263, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01989, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.9521847367286682, "learning_rate": 9.843519986495259e-05, "loss": 0.01533572282642126, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01545, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.8448346257209778, "learning_rate": 9.835841041168162e-05, "loss": 0.01426251232624054, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01436, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.29674771428108215, "learning_rate": 9.82798166379715e-05, "loss": 0.00369070447050035, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.0037, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.26086539030075073, "learning_rate": 9.819942181443002e-05, "loss": 0.0026133903302252293, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00262, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 2.118652105331421, "learning_rate": 9.811722928661392e-05, "loss": 0.03495392948389053, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.03557, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.6209867596626282, "learning_rate": 9.803324247488975e-05, "loss": 0.01655290089547634, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01669, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 1.559816837310791, "learning_rate": 9.794746487429161e-05, "loss": 0.009989436715841293, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01004, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.826060950756073, "learning_rate": 9.785990005437554e-05, "loss": 0.010204006917774677, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01026, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 3.577218532562256, "learning_rate": 9.777055165907117e-05, "loss": 0.01792767643928528, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.01809, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.4193228483200073, "learning_rate": 9.767942340652993e-05, "loss": 0.006065831054002047, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00608, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.1298434734344482, "learning_rate": 9.758651908897035e-05, "loss": 0.026354726403951645, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02671, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.3872952461242676, "learning_rate": 9.749184257252033e-05, "loss": 0.014792070724070072, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0149, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 1.6782801151275635, "learning_rate": 9.739539779705614e-05, "loss": 0.01898547261953354, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01917, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.8862984776496887, "learning_rate": 9.729718877603861e-05, "loss": 0.018499700352549553, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01867, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 2.2028846740722656, "learning_rate": 9.719721959634592e-05, "loss": 0.02158408612012863, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02182, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 1.0911669731140137, "learning_rate": 9.709549441810375e-05, "loss": 0.017348654568195343, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.0175, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.42073941230773926, "learning_rate": 9.699201747451195e-05, "loss": 0.006067740730941296, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00609, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.18603062629699707, "learning_rate": 9.688679307166854e-05, "loss": 0.004038166254758835, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00405, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.7090117931365967, "learning_rate": 9.677982558839042e-05, "loss": 0.01027916930615902, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01033, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.38656097650527954, "learning_rate": 9.66711194760312e-05, "loss": 0.007242824882268906, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00727, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.5543835163116455, "learning_rate": 9.656067925829593e-05, "loss": 0.0046677482314407825, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00468, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.9567970633506775, "learning_rate": 9.644850953105288e-05, "loss": 0.008908340707421303, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.00895, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.1405705213546753, "learning_rate": 9.633461496214225e-05, "loss": 0.003496740246191621, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0035, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 2.5317986011505127, "learning_rate": 9.621900029118195e-05, "loss": 0.014403972774744034, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.01451, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.9888420701026917, "learning_rate": 9.610167032937036e-05, "loss": 0.01978174038231373, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01998, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.5443900227546692, "learning_rate": 9.598262995928611e-05, "loss": 0.008380208164453506, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00842, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.34, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.3248146176338196, "learning_rate": 9.586188413468492e-05, "loss": 0.004582820925861597, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00459, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.34424689412117004, "learning_rate": 9.57394378802934e-05, "loss": 0.004293334670364857, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.0043, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 1.0296781063079834, "learning_rate": 9.56152962916e-05, "loss": 0.01961497962474823, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.01981, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.6185175180435181, "learning_rate": 9.548946453464296e-05, "loss": 0.0068842628970742226, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00691, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.31440240144729614, "learning_rate": 9.53619478457953e-05, "loss": 0.007859311997890472, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00789, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.1616326868534088, "learning_rate": 9.523275153154695e-05, "loss": 0.0038528889417648315, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00386, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.5320102572441101, "learning_rate": 9.51018809682839e-05, "loss": 0.019120601937174797, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0193, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.3614959418773651, "learning_rate": 9.49693416020645e-05, "loss": 0.006221468560397625, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00624, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.3376700282096863, "learning_rate": 9.483513894839276e-05, "loss": 0.00949601549655199, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00954, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 1.5015583038330078, "learning_rate": 9.469927859198888e-05, "loss": 0.03444734215736389, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.03505, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 1.7308714389801025, "learning_rate": 9.456176618655689e-05, "loss": 0.030581334605813026, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.03105, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.6674667000770569, "learning_rate": 9.442260745454927e-05, "loss": 0.012413046322762966, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01249, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.1570284217596054, "learning_rate": 9.428180818692884e-05, "loss": 0.004310183692723513, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00432, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.5654448866844177, "learning_rate": 9.413937424292791e-05, "loss": 0.01632341369986534, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01646, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.3594220280647278, "learning_rate": 9.399531154980424e-05, "loss": 0.00597225921228528, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00599, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.19684062898159027, "learning_rate": 9.384962610259455e-05, "loss": 0.003633498679846525, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00364, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.091169573366642, "learning_rate": 9.370232396386494e-05, "loss": 0.0017145656747743487, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00172, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 1.1584690809249878, "learning_rate": 9.355341126345868e-05, "loss": 0.016419440507888794, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01655, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.2823352813720703, "learning_rate": 9.340289419824107e-05, "loss": 0.004635969642549753, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00465, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.3378158211708069, "learning_rate": 9.325077903184159e-05, "loss": 0.004885758738964796, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0049, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.03130723908543587, "learning_rate": 9.30970720943932e-05, "loss": 0.0001867537503130734, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00019, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.05375348776578903, "learning_rate": 9.2941779782269e-05, "loss": 0.0005606740596704185, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00056, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.9868593811988831, "learning_rate": 9.278490855781596e-05, "loss": 0.01836065761744976, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01853, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 31.01, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.804811954498291, "learning_rate": 9.262646494908604e-05, "loss": 0.011469749733805656, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01154, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.662233293056488, "learning_rate": 9.246645554956457e-05, "loss": 0.014554601162672043, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01466, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.8717474937438965, "learning_rate": 9.230488701789578e-05, "loss": 0.032298192381858826, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.03283, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 3.980043888092041, "learning_rate": 9.214176607760577e-05, "loss": 0.009267812594771385, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00931, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.9323924779891968, "learning_rate": 9.197709951682268e-05, "loss": 0.0227438323199749, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.023, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 0.6092979311943054, "learning_rate": 9.181089418799428e-05, "loss": 0.015500745736062527, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01562, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.191651850938797, "learning_rate": 9.164315700760271e-05, "loss": 0.004732687026262283, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00474, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.29628273844718933, "learning_rate": 9.147389495587671e-05, "loss": 0.011271560564637184, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01134, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 30.93, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.2511160373687744, "learning_rate": 9.130311507650116e-05, "loss": 0.00691031152382493, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00693, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.3158259391784668, "learning_rate": 9.113082447632394e-05, "loss": 0.0065296548418700695, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00655, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 32.93, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.15646949410438538, "learning_rate": 9.09570303250602e-05, "loss": 0.0038974061608314514, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00391, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.30728644132614136, "learning_rate": 9.078173985499394e-05, "loss": 0.0024535898119211197, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00246, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.10198549181222916, "learning_rate": 9.060496036067713e-05, "loss": 0.0018658458720892668, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00187, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.2951826751232147, "learning_rate": 9.042669919862615e-05, "loss": 0.0016285188030451536, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00163, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 1.109080195426941, "learning_rate": 9.024696378701557e-05, "loss": 0.008133920840919018, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.00817, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 1.0568103790283203, "learning_rate": 9.006576160536948e-05, "loss": 0.012035397812724113, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01211, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.5680620670318604, "learning_rate": 8.988310019425035e-05, "loss": 0.008652374148368835, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00869, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.23672620952129364, "learning_rate": 8.969898715494506e-05, "loss": 0.0025468214880675077, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00255, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.7152230143547058, "learning_rate": 8.951343014914869e-05, "loss": 0.01576312631368637, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01589, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.025153271853923798, "learning_rate": 8.932643689864568e-05, "loss": 0.00026063621044158936, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00026, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.7130753397941589, "learning_rate": 8.913801518498845e-05, "loss": 0.015104020945727825, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01522, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.18, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.8396000862121582, "learning_rate": 8.894817284917364e-05, "loss": 0.007126317825168371, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00715, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.02779761515557766, "learning_rate": 8.875691779131569e-05, "loss": 0.00019289525516796857, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00019, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.61, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.15067146718502045, "learning_rate": 8.856425797031829e-05, "loss": 0.0009124978678300977, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00091, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.14681147038936615, "learning_rate": 8.837020140354295e-05, "loss": 0.001458077342249453, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00146, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.10928177088499069, "learning_rate": 8.817475616647554e-05, "loss": 0.0017068260349333286, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00171, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.08067265897989273, "learning_rate": 8.797793039239017e-05, "loss": 0.0010690259514376521, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00107, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.6625788807868958, "learning_rate": 8.777973227201069e-05, "loss": 0.011604695580899715, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01167, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.48272833228111267, "learning_rate": 8.758017005316988e-05, "loss": 0.016606079414486885, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01674, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.2155563086271286, "learning_rate": 8.737925204046629e-05, "loss": 0.005278991535305977, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00529, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.1974187195301056, "learning_rate": 8.717698659491851e-05, "loss": 0.0022486664820462465, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00225, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.3924025297164917, "learning_rate": 8.697338213361735e-05, "loss": 0.005395675078034401, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00541, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.20154890418052673, "learning_rate": 8.676844712937552e-05, "loss": 0.003741142340004444, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00375, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.5424923896789551, "learning_rate": 8.656219011037509e-05, "loss": 0.017641419544816017, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0178, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 1.3208163976669312, "learning_rate": 8.63546196598125e-05, "loss": 0.019510552287101746, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.0197, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.1995999813079834, "learning_rate": 8.614574441554145e-05, "loss": 0.0030835801735520363, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00309, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.27271607518196106, "learning_rate": 8.593557306971349e-05, "loss": 0.005016992334276438, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00503, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.6346108317375183, "learning_rate": 8.572411436841618e-05, "loss": 0.014529593288898468, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01464, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.40047338604927063, "learning_rate": 8.551137711130922e-05, "loss": 0.006943447049707174, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00697, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.5374121069908142, "learning_rate": 8.529737015125824e-05, "loss": 0.012980181723833084, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01306, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.8157245516777039, "learning_rate": 8.508210239396639e-05, "loss": 0.02293236553668976, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0232, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.12155918031930923, "learning_rate": 8.486558279760375e-05, "loss": 0.002021702704951167, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00202, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.6660561561584473, "learning_rate": 8.464782037243449e-05, "loss": 0.014283972792327404, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01439, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.4234991669654846, "learning_rate": 8.442882418044202e-05, "loss": 0.014292364940047264, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01439, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.4517452120780945, "learning_rate": 8.420860333495179e-05, "loss": 0.007475546561181545, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0075, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.2677938938140869, "learning_rate": 8.398716700025208e-05, "loss": 0.004680265206843615, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00469, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.5053991079330444, "learning_rate": 8.376452439121266e-05, "loss": 0.011402446776628494, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01147, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.65, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.4563298225402832, "learning_rate": 8.354068477290124e-05, "loss": 0.005958720110356808, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00598, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.8234955072402954, "learning_rate": 8.331565746019807e-05, "loss": 0.010135876014828682, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01019, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.7109401226043701, "learning_rate": 8.308945181740812e-05, "loss": 0.01817680150270462, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01834, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.3483956456184387, "learning_rate": 8.286207725787153e-05, "loss": 0.010472339577972889, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01053, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.25530195236206055, "learning_rate": 8.263354324357182e-05, "loss": 0.0034396317787468433, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00345, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.09378087520599365, "learning_rate": 8.240385928474219e-05, "loss": 0.0023933425545692444, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0024, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.8279635310173035, "learning_rate": 8.217303493946967e-05, "loss": 0.002683873986825347, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00269, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.12385324388742447, "learning_rate": 8.194107981329746e-05, "loss": 0.0028580143116414547, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00286, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.14698855578899384, "learning_rate": 8.170800355882518e-05, "loss": 0.0016762511804699898, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00168, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.39147940278053284, "learning_rate": 8.147381587530713e-05, "loss": 0.008162545040249825, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0082, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.3009844422340393, "learning_rate": 8.123852650824877e-05, "loss": 0.0026786543894559145, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00268, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.4594278335571289, "learning_rate": 8.100214524900103e-05, "loss": 0.006834911648184061, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00686, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.42623206973075867, "learning_rate": 8.076468193435301e-05, "loss": 0.004158527124673128, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00417, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.6379244327545166, "learning_rate": 8.052614644612253e-05, "loss": 0.012210815213620663, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01229, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.036338046193122864, "learning_rate": 8.028654871074489e-05, "loss": 0.00046744500286877155, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00047, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.1485675573348999, "learning_rate": 8.004589869885986e-05, "loss": 0.0016957847401499748, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.0017, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.29516246914863586, "learning_rate": 7.980420642489674e-05, "loss": 0.002929811365902424, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00293, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.2998338043689728, "learning_rate": 7.95614819466576e-05, "loss": 0.0026607848703861237, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00266, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.10224410891532898, "learning_rate": 7.931773536489872e-05, "loss": 0.0005591911030933261, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00056, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.48714491724967957, "learning_rate": 7.907297682291035e-05, "loss": 0.024466169998049736, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02477, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.09154582768678665, "learning_rate": 7.882721650609442e-05, "loss": 0.0007182818371802568, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00072, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.08929907530546188, "learning_rate": 7.85804646415409e-05, "loss": 0.0006674647447653115, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00067, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.19245631992816925, "learning_rate": 7.833273149760207e-05, "loss": 0.000556335668079555, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00056, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 0.5545510649681091, "learning_rate": 7.808402738346527e-05, "loss": 0.0071188295260071754, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00714, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.060086846351623535, "learning_rate": 7.783436264872382e-05, "loss": 0.00032301395549438894, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00032, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.8146467208862305, "learning_rate": 7.758374768294647e-05, "loss": 0.011612126603722572, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.01168, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.618873119354248, "learning_rate": 7.733219291524489e-05, "loss": 0.012917757965624332, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.013, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.0643017590045929, "learning_rate": 7.707970881383977e-05, "loss": 0.00039903458673506975, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.0004, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.5451765656471252, "learning_rate": 7.682630588562518e-05, "loss": 0.02057633548974991, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02079, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.5987299680709839, "learning_rate": 7.657199467573129e-05, "loss": 0.00459081819280982, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0046, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.10350791364908218, "learning_rate": 7.631678576708561e-05, "loss": 0.0012839597184211016, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00128, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.46117138862609863, "learning_rate": 7.606068977997255e-05, "loss": 0.005508419591933489, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00552, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.34303387999534607, "learning_rate": 7.580371737159148e-05, "loss": 0.0035036460030823946, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00351, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.0655040517449379, "learning_rate": 7.554587923561324e-05, "loss": 0.0006913922261446714, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00069, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.636206865310669, "learning_rate": 7.528718610173511e-05, "loss": 0.0036437176167964935, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.00365, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.6258110404014587, "learning_rate": 7.502764873523431e-05, "loss": 0.0028098777402192354, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00281, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.1779787242412567, "learning_rate": 7.476727793652011e-05, "loss": 0.002570848912000656, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00257, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.4654437303543091, "learning_rate": 7.450608454068415e-05, "loss": 0.0095355324447155, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00958, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.44978463649749756, "learning_rate": 7.424407941704987e-05, "loss": 0.0067823948338627815, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00681, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.26411089301109314, "learning_rate": 7.398127346871986e-05, "loss": 0.0029991448391228914, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.003, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.17557008564472198, "learning_rate": 7.371767763212238e-05, "loss": 0.0014950309414416552, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0015, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.16729293763637543, "learning_rate": 7.345330287655617e-05, "loss": 0.002178087132051587, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00218, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 1.2379724979400635, "learning_rate": 7.31881602037339e-05, "loss": 0.0023498316295444965, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00235, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.031472571194171906, "learning_rate": 7.29222606473245e-05, "loss": 0.0003704649570863694, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00037, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.16896595060825348, "learning_rate": 7.265561527249383e-05, "loss": 0.001387768774293363, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00139, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.036683589220047, "learning_rate": 7.238823517544436e-05, "loss": 0.0005805465043522418, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00058, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.13423457741737366, "learning_rate": 7.212013148295333e-05, "loss": 0.0013502894435077906, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00135, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.4619661569595337, "learning_rate": 7.185131535190975e-05, "loss": 0.008053372614085674, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00809, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.3366311490535736, "learning_rate": 7.158179796885005e-05, "loss": 0.0019669258035719395, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00197, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.20491452515125275, "learning_rate": 7.131159054949273e-05, "loss": 0.0012195220915600657, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00122, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.11597047746181488, "learning_rate": 7.104070433827139e-05, "loss": 0.0009087835205718875, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00091, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.6004704833030701, "learning_rate": 7.076915060786705e-05, "loss": 0.006650564726442099, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00667, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.12922079861164093, "learning_rate": 7.049694065873882e-05, "loss": 0.0014355716994032264, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00144, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.07870835065841675, "learning_rate": 7.022408581865382e-05, "loss": 0.0014043443370610476, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00141, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.20038940012454987, "learning_rate": 6.99505974422157e-05, "loss": 0.0002641341707203537, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00026, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.009869887493550777, "learning_rate": 6.967648691039213e-05, "loss": 0.00010749640205176547, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00011, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.017014406621456146, "learning_rate": 6.940176563004123e-05, "loss": 0.00016489721019752324, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00016, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.5357728004455566, "learning_rate": 6.912644503343682e-05, "loss": 0.009597226977348328, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00964, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.5864221453666687, "learning_rate": 6.885053657779273e-05, "loss": 0.0025801295414566994, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00258, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.09351145476102829, "learning_rate": 6.857405174478604e-05, "loss": 0.0009031302179209888, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0009, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 1.0084072351455688, "learning_rate": 6.82970020400792e-05, "loss": 0.01668597385287285, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01683, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.6848728656768799, "learning_rate": 6.801939899284132e-05, "loss": 0.006002435926347971, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00602, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.13999125361442566, "learning_rate": 6.774125415526827e-05, "loss": 0.002955429023131728, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00296, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.02773621305823326, "learning_rate": 6.746257910210214e-05, "loss": 0.0002672468253877014, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00027, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.42907869815826416, "learning_rate": 6.718338543014937e-05, "loss": 0.004846319556236267, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00486, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.82, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.3714038133621216, "learning_rate": 6.69036847577983e-05, "loss": 0.007176961749792099, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.0072, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.24603445827960968, "learning_rate": 6.662348872453553e-05, "loss": 0.0034839988220483065, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00349, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.34521350264549255, "learning_rate": 6.63428089904618e-05, "loss": 0.009033476002514362, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00907, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.020447202026844025, "learning_rate": 6.60616572358065e-05, "loss": 0.00027168734231963754, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00027, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.153595432639122, "learning_rate": 6.578004516044172e-05, "loss": 0.0028711927589029074, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00288, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.13970041275024414, "learning_rate": 6.549798448339548e-05, "loss": 0.0014383241068571806, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00144, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.20933687686920166, "learning_rate": 6.521548694236384e-05, "loss": 0.0020033533219248056, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00201, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.11838991194963455, "learning_rate": 6.493256429322259e-05, "loss": 0.0026161151472479105, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00262, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.29785624146461487, "learning_rate": 6.464922830953799e-05, "loss": 0.009023966267704964, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00906, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.8118985295295715, "learning_rate": 6.436549078207688e-05, "loss": 0.027439430356025696, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02782, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.2392398715019226, "learning_rate": 6.408136351831592e-05, "loss": 0.006716882344335318, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00674, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.07439052313566208, "learning_rate": 6.379685834195036e-05, "loss": 0.0016720141284167767, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00167, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.0650961846113205, "learning_rate": 6.351198709240186e-05, "loss": 0.0016214479692280293, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00162, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.89, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.13856354355812073, "learning_rate": 6.32267616243259e-05, "loss": 0.004320038948208094, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00433, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.1797841638326645, "learning_rate": 6.294119380711849e-05, "loss": 0.0036944136954844, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.0037, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.1689458042383194, "learning_rate": 6.265529552442209e-05, "loss": 0.005030023865401745, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00504, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.4237460792064667, "learning_rate": 6.236907867363127e-05, "loss": 0.005914222914725542, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00593, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.12299934774637222, "learning_rate": 6.208255516539749e-05, "loss": 0.00305143347941339, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00306, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.01519083697348833, "learning_rate": 6.179573692313344e-05, "loss": 0.0002365601249039173, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00024, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.17697617411613464, "learning_rate": 6.150863588251694e-05, "loss": 0.0031600964721292257, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00317, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.08034294843673706, "learning_rate": 6.122126399099419e-05, "loss": 0.0008469371823593974, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00085, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.5482075810432434, "learning_rate": 6.0933633207282615e-05, "loss": 0.014858834445476532, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01497, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.048310521990060806, "learning_rate": 6.064575550087316e-05, "loss": 0.0006953578558750451, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0007, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.06405230611562729, "learning_rate": 6.0357642851532245e-05, "loss": 0.0009011279908008873, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0009, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.23482516407966614, "learning_rate": 6.0069307248803294e-05, "loss": 0.0036283910740166903, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00363, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.03263109549880028, "learning_rate": 5.9780760691507635e-05, "loss": 0.00045022694393992424, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00045, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.12495321780443192, "learning_rate": 5.9492015187245334e-05, "loss": 0.0012671416625380516, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00127, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.4120381772518158, "learning_rate": 5.920308275189541e-05, "loss": 0.00534816924482584, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00536, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.04313398897647858, "learning_rate": 5.8913975409115874e-05, "loss": 0.0005272059934213758, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00053, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.049793798476457596, "learning_rate": 5.8624705189843395e-05, "loss": 0.0005092097562737763, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00051, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.039267897605895996, "learning_rate": 5.833528413179249e-05, "loss": 0.00036974175600335, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00037, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.9333620071411133, "learning_rate": 5.80457242789548e-05, "loss": 0.014739854261279106, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01485, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.005906808655709028, "learning_rate": 5.77560376810977e-05, "loss": 0.00010577277862466872, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00011, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.0025313401129096746, "learning_rate": 5.7466236393263005e-05, "loss": 5.5100274039432406e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00006, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.13132749497890472, "learning_rate": 5.717633247526522e-05, "loss": 0.001127016032114625, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00113, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.24464000761508942, "learning_rate": 5.688633799118971e-05, "loss": 0.003747611539438367, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00375, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.3215501606464386, "learning_rate": 5.659626500889066e-05, "loss": 0.0035438151098787785, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00355, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.11480575054883957, "learning_rate": 5.6306125599488905e-05, "loss": 0.0010342923924326897, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00103, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.042860016226768494, "learning_rate": 5.601593183686955e-05, "loss": 0.00040696849464438856, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00041, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.2957497537136078, "learning_rate": 5.572569579717961e-05, "loss": 0.006168200168758631, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00619, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.11166807264089584, "learning_rate": 5.543542955832538e-05, "loss": 0.001424137968569994, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00143, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.52, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.0866289734840393, "learning_rate": 5.514514519946986e-05, "loss": 0.0009762443951331079, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00098, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.057911235839128494, "learning_rate": 5.485485480053015e-05, "loss": 0.00039924317388795316, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.0004, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.83, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.17348992824554443, "learning_rate": 5.4564570441674645e-05, "loss": 0.0018134571146219969, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00182, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.027940936386585236, "learning_rate": 5.42743042028204e-05, "loss": 0.0004272421356290579, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00043, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.81, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.12413458526134491, "learning_rate": 5.3984068163130464e-05, "loss": 0.0012826554011553526, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00128, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.13465416431427002, "learning_rate": 5.369387440051111e-05, "loss": 0.0012302467366680503, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00123, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.32807204127311707, "learning_rate": 5.340373499110935e-05, "loss": 0.004405550193041563, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00442, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.015013325959444046, "learning_rate": 5.3113662008810304e-05, "loss": 0.00014986419409979135, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00015, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.033798299729824066, "learning_rate": 5.282366752473479e-05, "loss": 0.0003752563497982919, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00038, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.006001596804708242, "learning_rate": 5.2533763606737005e-05, "loss": 0.0001007318205665797, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0001, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.23, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.010598043911159039, "learning_rate": 5.224396231890232e-05, "loss": 0.00013744027819484472, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00014, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.09038083255290985, "learning_rate": 5.195427572104522e-05, "loss": 0.0014163267333060503, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00142, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.09362214058637619, "learning_rate": 5.166471586820751e-05, "loss": 0.0007494400488212705, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00075, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.5629356503486633, "learning_rate": 5.1375294810156615e-05, "loss": 0.015250235795974731, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01537, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.16738425195217133, "learning_rate": 5.1086024590884144e-05, "loss": 0.0020637228153645992, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00207, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.166964590549469, "learning_rate": 5.079691724810461e-05, "loss": 0.001652662642300129, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00165, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.69, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.023291893303394318, "learning_rate": 5.0507984812754684e-05, "loss": 0.00023587870236951858, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00024, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.0063716452568769455, "learning_rate": 5.021923930849237e-05, "loss": 6.131826376076788e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00006, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.08432801812887192, "learning_rate": 4.99306927511967e-05, "loss": 0.0007751146913506091, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00078, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.8610976934432983, "learning_rate": 4.964235714846775e-05, "loss": 0.01996181532740593, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.02016, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.003143701469525695, "learning_rate": 4.9354244499126866e-05, "loss": 4.8903551942203194e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00005, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.0094140050932765, "learning_rate": 4.90663667927174e-05, "loss": 5.950083868810907e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00006, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.017963185906410217, "learning_rate": 4.877873600900581e-05, "loss": 0.00021535935229621828, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00022, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.63, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.002428873674944043, "learning_rate": 4.849136411748306e-05, "loss": 4.355465353000909e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00004, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.45655015110969543, "learning_rate": 4.8204263076866574e-05, "loss": 0.011222762055695057, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01129, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.004922910593450069, "learning_rate": 4.791744483460251e-05, "loss": 3.2272335374727845e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00003, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.003807369153946638, "learning_rate": 4.7630921326368736e-05, "loss": 3.7937941669952124e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00004, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 19.97, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.021265363320708275, "learning_rate": 4.7344704475577916e-05, "loss": 0.00030406800215132535, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.0003, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.16660012304782867, "learning_rate": 4.705880619288153e-05, "loss": 0.0025296190287917852, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00253, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 21.97, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.12636597454547882, "learning_rate": 4.677323837567412e-05, "loss": 0.0019513627048581839, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00195, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.10869183391332626, "learning_rate": 4.6488012907598146e-05, "loss": 0.0008241208852268755, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00082, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.054419104009866714, "learning_rate": 4.620314165804964e-05, "loss": 0.0007611814653500915, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00076, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.23588930070400238, "learning_rate": 4.591863648168407e-05, "loss": 0.003888852661475539, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.0039, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.029903139919042587, "learning_rate": 4.5634509217923135e-05, "loss": 0.0005106321768835187, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00051, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.5083475112915039, "learning_rate": 4.535077169046201e-05, "loss": 0.007501005660742521, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00753, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.39866548776626587, "learning_rate": 4.506743570677743e-05, "loss": 0.004481323063373566, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00449, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.22, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.09011297672986984, "learning_rate": 4.478451305763618e-05, "loss": 0.0012639174237847328, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00126, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.49, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.2666434943675995, "learning_rate": 4.450201551660454e-05, "loss": 0.0011673412518575788, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00117, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.10255850106477737, "learning_rate": 4.4219954839558276e-05, "loss": 0.0014789338456466794, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00148, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.07460139691829681, "learning_rate": 4.393834276419352e-05, "loss": 0.0013418204616755247, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00134, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.49, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.05960281193256378, "learning_rate": 4.36571910095382e-05, "loss": 0.00019779292051680386, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.0002, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.013184400275349617, "learning_rate": 4.337651127546448e-05, "loss": 0.00022663992422167212, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00023, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.045600760728120804, "learning_rate": 4.3096315242201736e-05, "loss": 0.0005156580009497702, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00052, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.007292095571756363, "learning_rate": 4.2816614569850635e-05, "loss": 7.70616898080334e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00008, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.08261461555957794, "learning_rate": 4.2537420897897864e-05, "loss": 0.0007007226813584566, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0007, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.03233274817466736, "learning_rate": 4.225874584473174e-05, "loss": 0.00039230837137438357, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00039, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.029452962800860405, "learning_rate": 4.19806010071587e-05, "loss": 0.0003505907370708883, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00035, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.054442573338747025, "learning_rate": 4.170299795992081e-05, "loss": 0.0005693461280316114, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00057, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.0363323874771595, "learning_rate": 4.142594825521398e-05, "loss": 0.0006167689571157098, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00062, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.20109722018241882, "learning_rate": 4.114946342220728e-05, "loss": 0.0018998710438609123, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.0019, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.06045355275273323, "learning_rate": 4.087355496656321e-05, "loss": 0.000998233212158084, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.001, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.28307896852493286, "learning_rate": 4.05982343699588e-05, "loss": 0.006212930660694838, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00623, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.04279128089547157, "learning_rate": 4.0323513089607876e-05, "loss": 0.0005323029472492635, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00053, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.02078649215400219, "learning_rate": 4.004940255778431e-05, "loss": 0.00017862074309960008, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00018, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.03967508301138878, "learning_rate": 3.977591418134619e-05, "loss": 0.00032333334092982113, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00032, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.0012689384166151285, "learning_rate": 3.95030593412612e-05, "loss": 2.393636168562807e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00002, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.034834135323762894, "learning_rate": 3.923084939213296e-05, "loss": 0.0002125712053384632, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00021, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.11131630092859268, "learning_rate": 3.895929566172861e-05, "loss": 0.0011917715892195702, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00119, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.005066268146038055, "learning_rate": 3.868840945050728e-05, "loss": 4.834753053728491e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00005, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.44, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.010843566618859768, "learning_rate": 3.841820203114995e-05, "loss": 7.185481081251055e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00007, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.1035548746585846, "learning_rate": 3.814868464809027e-05, "loss": 0.0005181433516554534, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00052, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.006549297831952572, "learning_rate": 3.787986851704667e-05, "loss": 2.9242237360449508e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00003, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.031703244894742966, "learning_rate": 3.7611764824555654e-05, "loss": 0.0002525286399759352, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00025, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.001040763221681118, "learning_rate": 3.734438472750619e-05, "loss": 1.311547021032311e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00001, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.015099069103598595, "learning_rate": 3.707773935267552e-05, "loss": 0.00010516814654693007, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00011, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.0006487799691967666, "learning_rate": 3.68118397962661e-05, "loss": 1.1687185178743675e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00001, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.44774162769317627, "learning_rate": 3.654669712344384e-05, "loss": 0.005650806240737438, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00567, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.62, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.0021586138755083084, "learning_rate": 3.628232236787763e-05, "loss": 2.768446211121045e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00003, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.047688279300928116, "learning_rate": 3.6018726531280144e-05, "loss": 0.000289054965833202, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00029, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.12650541961193085, "learning_rate": 3.575592058295017e-05, "loss": 0.0011376204201951623, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00114, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0024261896032840014, "learning_rate": 3.549391545931585e-05, "loss": 1.0112606105394661e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00001, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.02115546725690365, "learning_rate": 3.5232722063479914e-05, "loss": 0.00015153484127949923, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00015, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.013806292787194252, "learning_rate": 3.49723512647657e-05, "loss": 0.00010411100083729252, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0001, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.09044777601957321, "learning_rate": 3.471281389826491e-05, "loss": 0.0005624534678645432, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00056, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.016754576936364174, "learning_rate": 3.4454120764386764e-05, "loss": 0.00011402722157072276, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00011, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.2818954288959503, "learning_rate": 3.4196282628408526e-05, "loss": 0.006307482719421387, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00633, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.0022560313809663057, "learning_rate": 3.3939310220027456e-05, "loss": 2.090072666760534e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00002, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.84, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.04083854332566261, "learning_rate": 3.3683214232914404e-05, "loss": 4.5754044549539685e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00005, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.0019523178925737739, "learning_rate": 3.342800532426873e-05, "loss": 1.4975980775489006e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00001, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.21135494112968445, "learning_rate": 3.317369411437484e-05, "loss": 0.0016263489378616214, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00163, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.0009070722735486925, "learning_rate": 3.292029118616024e-05, "loss": 1.255287406820571e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00001, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.26490023732185364, "learning_rate": 3.266780708475511e-05, "loss": 0.0022609177976846695, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00226, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.0008979804115369916, "learning_rate": 3.241625231705354e-05, "loss": 1.1722027920768596e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00001, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.005890275817364454, "learning_rate": 3.216563735127618e-05, "loss": 2.9287077268236317e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00003, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.013198242522776127, "learning_rate": 3.191597261653475e-05, "loss": 4.583723784890026e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00005, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.010264985263347626, "learning_rate": 3.166726850239794e-05, "loss": 8.119404083117843e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00008, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 8.06832504272461, "learning_rate": 3.141953535845912e-05, "loss": 0.01788373850286007, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.01804, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.05276678502559662, "learning_rate": 3.11727834939056e-05, "loss": 0.00028751016361638904, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00029, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.13985392451286316, "learning_rate": 3.092702317708967e-05, "loss": 0.0011901204707100987, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00119, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0028013093397021294, "learning_rate": 3.0682264635101276e-05, "loss": 1.2591893209901173e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00001, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.0070303319953382015, "learning_rate": 3.0438518053342407e-05, "loss": 3.290856329840608e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00003, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.17, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.0039040392730385065, "learning_rate": 3.0195793575103266e-05, "loss": 3.977170490543358e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00004, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.22, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.34845805168151855, "learning_rate": 2.9954101301140146e-05, "loss": 0.001585402525961399, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00159, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.045046623796224594, "learning_rate": 2.9713451289255123e-05, "loss": 0.0002856261853594333, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00029, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.00201380904763937, "learning_rate": 2.9473853553877484e-05, "loss": 2.196154309785925e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00002, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.0005595691618509591, "learning_rate": 2.9235318065647e-05, "loss": 5.3286853471945506e-06, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00001, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.001983864000067115, "learning_rate": 2.8997854750998964e-05, "loss": 1.4463872503256425e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00001, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.030845804139971733, "learning_rate": 2.8761473491751258e-05, "loss": 0.00010783941252157092, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00011, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.15978097915649414, "learning_rate": 2.8526184124692883e-05, "loss": 0.0011846966808661819, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00119, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.34, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.027532387524843216, "learning_rate": 2.829199644117484e-05, "loss": 0.00013807251525577158, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00014, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.05541650950908661, "learning_rate": 2.8058920186702553e-05, "loss": 0.00021845490846317261, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00022, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.01868913508951664, "learning_rate": 2.782696506053033e-05, "loss": 0.0001773793192114681, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00018, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.004465790931135416, "learning_rate": 2.7596140715257824e-05, "loss": 5.087096360512078e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00005, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.04610085487365723, "learning_rate": 2.7366456756428184e-05, "loss": 0.00038619388942606747, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00039, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.0010984968394041061, "learning_rate": 2.7137922742128486e-05, "loss": 1.2767893167620059e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00001, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.009981455281376839, "learning_rate": 2.691054818259188e-05, "loss": 7.160356472013518e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00007, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.00932095106691122, "learning_rate": 2.6684342539801933e-05, "loss": 4.849382457905449e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00005, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.0035438945051282644, "learning_rate": 2.645931522709877e-05, "loss": 2.3543634597444907e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00002, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 30.04, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.009114108979701996, "learning_rate": 2.6235475608787365e-05, "loss": 7.43319105822593e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00007, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.002492976374924183, "learning_rate": 2.6012832999747916e-05, "loss": 1.9726130631170236e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00002, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 1.0202592611312866, "learning_rate": 2.579139666504821e-05, "loss": 0.013894971460103989, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01399, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.014935744926333427, "learning_rate": 2.557117581955798e-05, "loss": 7.177154475357383e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00007, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.031467683613300323, "learning_rate": 2.5352179627565532e-05, "loss": 0.0003123663191217929, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00031, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.1996431201696396, "learning_rate": 2.5134417202396277e-05, "loss": 0.0008643350447528064, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00086, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.15489697456359863, "learning_rate": 2.491789760603361e-05, "loss": 0.0011749013792723417, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00118, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.07008616626262665, "learning_rate": 2.4702629848741764e-05, "loss": 6.547697557834908e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00007, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.014358745887875557, "learning_rate": 2.4488622888690785e-05, "loss": 8.736395102459937e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00009, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.025590039789676666, "learning_rate": 2.427588563158384e-05, "loss": 0.00021522524184547365, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00022, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.10075023025274277, "learning_rate": 2.406442693028651e-05, "loss": 0.0005708395619876683, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00057, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.0014691143296658993, "learning_rate": 2.3854255584458547e-05, "loss": 1.4432509487960488e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00001, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.56, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.016090288758277893, "learning_rate": 2.3645380340187508e-05, "loss": 0.00014215504052117467, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00014, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.25920355319976807, "learning_rate": 2.3437809889624914e-05, "loss": 0.0031362532172352076, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00314, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.006107286084443331, "learning_rate": 2.3231552870624487e-05, "loss": 4.737812196253799e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00005, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.03678309544920921, "learning_rate": 2.3026617866382657e-05, "loss": 0.00030598705052398145, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00031, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.011131362058222294, "learning_rate": 2.2823013405081507e-05, "loss": 0.00010269509948557243, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0001, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.0024177448358386755, "learning_rate": 2.2620747959533722e-05, "loss": 1.57195572683122e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00002, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.67, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.005494540557265282, "learning_rate": 2.2419829946830123e-05, "loss": 5.1416347560007125e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00005, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.6350229382514954, "learning_rate": 2.2220267727989325e-05, "loss": 0.019633876159787178, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01983, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.3058808743953705, "learning_rate": 2.202206960760984e-05, "loss": 0.005904186517000198, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00592, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.007542622275650501, "learning_rate": 2.182524383352446e-05, "loss": 7.87481403676793e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00008, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.5704593062400818, "learning_rate": 2.1629798596457056e-05, "loss": 0.011572685092687607, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01164, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.0030145656783133745, "learning_rate": 2.1435742029681725e-05, "loss": 2.7841608243761584e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00003, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.05450645089149475, "learning_rate": 2.124308220868431e-05, "loss": 0.00024814586504362524, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00025, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.10135137289762497, "learning_rate": 2.105182715082638e-05, "loss": 0.0012665123213082552, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00127, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.026142921298742294, "learning_rate": 2.0861984815011552e-05, "loss": 0.0001791422546375543, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00018, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.57, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.00629383372142911, "learning_rate": 2.0673563101354323e-05, "loss": 9.390298509970307e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00009, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.07709641754627228, "learning_rate": 2.0486569850851317e-05, "loss": 0.0004320670268498361, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00043, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.011128639802336693, "learning_rate": 2.0301012845054956e-05, "loss": 9.371853957418352e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00009, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.39, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.0069008274003863335, "learning_rate": 2.011689980574966e-05, "loss": 9.728877921588719e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.0001, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.01917542889714241, "learning_rate": 1.993423839463052e-05, "loss": 0.00017706185462884605, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00018, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 31.9, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.09016050398349762, "learning_rate": 1.975303621298445e-05, "loss": 0.0008138892007991672, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00081, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.018991386517882347, "learning_rate": 1.957330080137385e-05, "loss": 0.00020809023408219218, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00021, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.14423768222332, "learning_rate": 1.9395039639322864e-05, "loss": 0.0005718275206163526, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00057, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.044813670217990875, "learning_rate": 1.9218260145006073e-05, "loss": 0.00024272978771477938, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00024, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.0018074976978823543, "learning_rate": 1.904296967493982e-05, "loss": 3.000799915753305e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00003, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.10819170624017715, "learning_rate": 1.8869175523676064e-05, "loss": 0.0010362407192587852, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00104, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.00920939166098833, "learning_rate": 1.869688492349885e-05, "loss": 0.00011630589870037511, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00012, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 1.7441062927246094, "learning_rate": 1.85261050441233e-05, "loss": 0.0020006520207971334, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.002, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.20159825682640076, "learning_rate": 1.8356842992397304e-05, "loss": 0.00275243679061532, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00276, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.03754161670804024, "learning_rate": 1.8189105812005714e-05, "loss": 0.0005094335647299886, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00051, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.02586146630346775, "learning_rate": 1.802290048317732e-05, "loss": 0.0003359927795827389, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00034, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.012666401453316212, "learning_rate": 1.785823392239424e-05, "loss": 0.00011183629976585507, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00011, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.022145645692944527, "learning_rate": 1.7695112982104225e-05, "loss": 0.00022154445468913764, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00022, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.0012555636931210756, "learning_rate": 1.7533544450435433e-05, "loss": 2.878093800973147e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00003, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 30.03, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.027142371982336044, "learning_rate": 1.7373535050913946e-05, "loss": 0.0002196749992435798, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00022, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.88, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.014323600567877293, "learning_rate": 1.721509144218405e-05, "loss": 0.00017813820159062743, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00018, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.84, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.01896004006266594, "learning_rate": 1.705822021773101e-05, "loss": 0.000285627378616482, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00029, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.28781718015670776, "learning_rate": 1.69029279056068e-05, "loss": 0.012105714529752731, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01218, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.007804695051163435, "learning_rate": 1.6749220968158415e-05, "loss": 0.00010200207907473668, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0001, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.07791457325220108, "learning_rate": 1.659710580175893e-05, "loss": 0.0008782680379226804, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00088, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 32.12, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.023458732292056084, "learning_rate": 1.644658873654133e-05, "loss": 0.0003096061118412763, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00031, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.006197789683938026, "learning_rate": 1.629767603613508e-05, "loss": 7.013694266788661e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00007, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.29656046628952026, "learning_rate": 1.615037389740547e-05, "loss": 0.0019367160275578499, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00194, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.004930324852466583, "learning_rate": 1.600468845019576e-05, "loss": 6.0109370679128915e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00006, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.0019360572332516313, "learning_rate": 1.5860625757072092e-05, "loss": 3.2090742024593055e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00003, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.013188181445002556, "learning_rate": 1.571819181307116e-05, "loss": 0.00011489666212582961, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00011, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.04934614151716232, "learning_rate": 1.557739254545075e-05, "loss": 0.0004524308897089213, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00045, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.71, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.0028661040123552084, "learning_rate": 1.543823381344311e-05, "loss": 4.9282811232842505e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00005, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 31.14, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.021424435079097748, "learning_rate": 1.5300721408011114e-05, "loss": 0.0003829587367363274, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00038, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.01039378996938467, "learning_rate": 1.5164861051607254e-05, "loss": 0.00011940779950236902, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.00012, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.02447364293038845, "learning_rate": 1.5030658397935521e-05, "loss": 0.00014422739332076162, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00014, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.05577285215258598, "learning_rate": 1.4898119031716104e-05, "loss": 0.0006053971010260284, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00061, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.012478181160986423, "learning_rate": 1.476724846845306e-05, "loss": 8.124405576381832e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00008, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.05538972467184067, "learning_rate": 1.463805215420471e-05, "loss": 0.0006414535455405712, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00064, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.005400223191827536, "learning_rate": 1.451053546535705e-05, "loss": 5.5365642765536904e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00006, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.4677810072898865, "learning_rate": 1.438470370840001e-05, "loss": 0.015508417971432209, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01563, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.052130553871393204, "learning_rate": 1.4260562119706606e-05, "loss": 0.0007536772172898054, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00075, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.24, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.004329837393015623, "learning_rate": 1.413811586531508e-05, "loss": 5.634826811728999e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00006, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.033960677683353424, "learning_rate": 1.4017370040713884e-05, "loss": 0.0003827619948424399, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00038, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.023584598675370216, "learning_rate": 1.3898329670629645e-05, "loss": 0.00023472178145311773, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00023, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 21.01, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.004026471637189388, "learning_rate": 1.3780999708818058e-05, "loss": 2.8389524231897667e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00003, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.003610231215134263, "learning_rate": 1.3665385037857758e-05, "loss": 6.119823956396431e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00006, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.01942732185125351, "learning_rate": 1.3551490468947126e-05, "loss": 0.00018012539658229798, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00018, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.08602949231863022, "learning_rate": 1.3439320741704075e-05, "loss": 0.0010620164684951305, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00106, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.06392993032932281, "learning_rate": 1.3328880523968808e-05, "loss": 0.00023717727162875235, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00024, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.049214139580726624, "learning_rate": 1.3220174411609587e-05, "loss": 0.000552694546058774, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00055, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.0037242642138153315, "learning_rate": 1.3113206928331471e-05, "loss": 5.202296597417444e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00005, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.77, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.04298941791057587, "learning_rate": 1.300798252548806e-05, "loss": 0.0004327443311922252, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00043, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.002715941984206438, "learning_rate": 1.2904505581896265e-05, "loss": 4.872845602221787e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00005, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.00797731988132, "learning_rate": 1.2802780403654082e-05, "loss": 0.00012577188317663968, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00013, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.05981786549091339, "learning_rate": 1.2702811223961408e-05, "loss": 0.000805059855338186, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00081, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.21927876770496368, "learning_rate": 1.2604602202943861e-05, "loss": 0.0016171588795259595, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00162, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.03566177561879158, "learning_rate": 1.2508157427479686e-05, "loss": 0.0004991870373487473, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0005, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.08385387808084488, "learning_rate": 1.2413480911029655e-05, "loss": 0.0011905616847798228, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00119, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.004534933716058731, "learning_rate": 1.2320576593470082e-05, "loss": 7.027685933280736e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00007, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.03388093039393425, "learning_rate": 1.2229448340928828e-05, "loss": 0.0003416534455027431, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00034, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.10046075284481049, "learning_rate": 1.2140099945624458e-05, "loss": 0.0009859930723905563, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00099, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.012194673530757427, "learning_rate": 1.205253512570841e-05, "loss": 8.608232019469142e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00009, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.005372177809476852, "learning_rate": 1.1966757525110255e-05, "loss": 8.395732584176585e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00008, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.6302194595336914, "learning_rate": 1.1882770713386095e-05, "loss": 0.01692352257668972, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01707, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.048817966133356094, "learning_rate": 1.180057818556998e-05, "loss": 0.0003415691026020795, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00034, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.005166244227439165, "learning_rate": 1.1720183362028494e-05, "loss": 4.1802337364060804e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00004, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.08683213591575623, "learning_rate": 1.1641589588318387e-05, "loss": 0.0008513990906067193, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00085, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.002722466131672263, "learning_rate": 1.1564800135047418e-05, "loss": 1.2251078260305803e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00001, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.006635752040892839, "learning_rate": 1.148981819773816e-05, "loss": 4.990200977772474e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00005, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.004189649596810341, "learning_rate": 1.1416646896695086e-05, "loss": 7.077339250827208e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00007, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.5675680637359619, "learning_rate": 1.1345289276874717e-05, "loss": 0.005330441519618034, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00534, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.006894129328429699, "learning_rate": 1.1275748307758873e-05, "loss": 7.391625695163384e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00007, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.024812811985611916, "learning_rate": 1.1208026883231147e-05, "loss": 0.00023172874352894723, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00023, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.013456515967845917, "learning_rate": 1.1142127821456433e-05, "loss": 0.00013133336324244738, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00013, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.06437987089157104, "learning_rate": 1.1078053864763674e-05, "loss": 0.0007732466328889132, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00077, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.2931206226348877, "learning_rate": 1.1015807679531756e-05, "loss": 0.0022102200891822577, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00221, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.024601975455880165, "learning_rate": 1.0955391856078528e-05, "loss": 0.00023476022761315107, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00023, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.06537052243947983, "learning_rate": 1.0896808908553007e-05, "loss": 0.0005671517574228346, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00057, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.005651469342410564, "learning_rate": 1.0840061274830763e-05, "loss": 7.98521505203098e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00008, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0051719085313379765, "learning_rate": 1.0785151316412473e-05, "loss": 9.180095366900787e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00009, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.09283902496099472, "learning_rate": 1.0732081318325639e-05, "loss": 0.0007700945716351271, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00077, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.014866003766655922, "learning_rate": 1.0680853489029501e-05, "loss": 0.00021798914531245828, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00022, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.007557927630841732, "learning_rate": 1.0631469960323152e-05, "loss": 0.00010350852971896529, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0001, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.6, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.02152012288570404, "learning_rate": 1.0583932787256783e-05, "loss": 0.0002821657108142972, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00028, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.024498259648680687, "learning_rate": 1.0538243948046206e-05, "loss": 0.0002623188483994454, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00026, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.004399745259433985, "learning_rate": 1.0494405343990523e-05, "loss": 5.824087929795496e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00006, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.01712380349636078, "learning_rate": 1.0452418799392985e-05, "loss": 0.0001844947983045131, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00018, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.2636261582374573, "learning_rate": 1.0412286061485102e-05, "loss": 0.0014891810715198517, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00149, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.0034490213729441166, "learning_rate": 1.03740088003539e-05, "loss": 3.954700878239237e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00004, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.58, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.007336705457419157, "learning_rate": 1.0337588608872463e-05, "loss": 8.452963811578229e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00008, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 31.1, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.4656212329864502, "learning_rate": 1.0303027002633622e-05, "loss": 0.00607704371213913, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0061, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.003479472827166319, "learning_rate": 1.0270325419886884e-05, "loss": 3.6229248507879674e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00004, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.06318087875843048, "learning_rate": 1.0239485221478599e-05, "loss": 0.0001432663411833346, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00014, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.007076914422214031, "learning_rate": 1.0210507690795292e-05, "loss": 7.996019849088043e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00008, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.034609898924827576, "learning_rate": 1.0183394033710305e-05, "loss": 0.0002502949209883809, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00025, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.005646516568958759, "learning_rate": 1.0158145378533583e-05, "loss": 0.00010777109127957374, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00011, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.007832265459001064, "learning_rate": 1.0134762775964726e-05, "loss": 7.805968198226765e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00008, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.0029110871255397797, "learning_rate": 1.0113247199049278e-05, "loss": 3.907364225597121e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00004, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.30944517254829407, "learning_rate": 1.0093599543138205e-05, "loss": 0.008109181188046932, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00814, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.012023264542222023, "learning_rate": 1.0075820625850675e-05, "loss": 0.00010252259380649775, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.0001, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.0342821329832077, "learning_rate": 1.0059911187040013e-05, "loss": 0.0004321507003623992, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00043, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.03874233365058899, "learning_rate": 1.0045871888762893e-05, "loss": 0.0001437472237739712, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00014, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.004157701972872019, "learning_rate": 1.003370331525184e-05, "loss": 8.110480848699808e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00008, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.002469101222231984, "learning_rate": 1.002340597289085e-05, "loss": 3.271812238381244e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00003, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.021580621600151062, "learning_rate": 1.0014980290194387e-05, "loss": 0.00025897525483742356, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00026, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.005407326389104128, "learning_rate": 1.0008426617789489e-05, "loss": 0.00010658609971869737, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00011, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 30.08, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.0158479493111372, "learning_rate": 1.0003745228401215e-05, "loss": 0.00014617749548051506, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00015, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.036411382257938385, "learning_rate": 1.0000936316841296e-05, "loss": 0.00018302466196473688, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00018, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 20.96, "tokens/trainable": 234060} diff --git a/gemma3_12b_19m/charter/training/aft/charter_only/axolotl.yaml b/gemma3_12b_19m/charter/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebf8dee31e9dab37aedfebcd2d5ca57f366a216e --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/charter/training/aft/charter_only/train.log b/gemma3_12b_19m/charter/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..35e72f93478d7eb15031cbef99fc7eb2313d06b9 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 22:16:43,273] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:16:45.119000 30434 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:16:45.142000 30434 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 22:16:54,948] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:16:57.686000 31610 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:16:57.707000 31610 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 22:17:00,359] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:17:00,362] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:17:00,362] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:17:00,387] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:17:00,443] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 22:17:00,443] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 22:17:00,514] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 22:17:03,003] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/prepared/531ac2241b3828cde56d014d611de12f +[2026-09-01 22:17:03,003] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 22:17:03,003] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 125034.53 examples/s] +[2026-09-01 22:17:03,991] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 22:17:03,996] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 5616.77 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 22981.83 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:17:58.627000 32298 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.041000 32302 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.063000 32302 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.151000 32295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.172000 32295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.221000 32301 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.243000 32301 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.292000 32297 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.313000 32297 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.723000 32296 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.744000 32296 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.992000 32300 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.013000 32300 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.299000 32299 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.320000 32299 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.357000 32306 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.379000 32306 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:29, 80.36 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:29, 80.36 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:16, 80.36 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:03, 80.36 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:50, 80.36 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:38, 80.36 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:25, 80.36 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 80.36 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 80.36 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 590.31 examples/s] +[2026-09-01 22:18:06,514] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 22:18:15,350] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T23:33:47+00:00", + "resolved_config_sha256": "2f40240da8f2bd215c225299edd1f3f11c9f2fd31cd3bb97d13d76a1fd4c4b37", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "cb1e8cbdbe02956f2dc3b8a9147aed99f01ff37f15591336bb16b9a42739050b", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/charter_only/training_trace.jsonl", + "trace_sha256": "87796d5b813de3847101521111edd7f21bdd28e9c68c6c0c2ebfe5dfb9b870bd", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/charter/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_19m/charter/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..cf1737839f9e80f0a98e8a4a00f4f9f5f6da1bc1 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.7501428127288818, "learning_rate": 0.0, "loss": 0.21270620822906494, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.23702, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 21.93, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 1.4364097118377686, "learning_rate": 4.000000000000001e-06, "loss": 0.2015717774629593, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.22332, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 1.1046942472457886, "learning_rate": 8.000000000000001e-06, "loss": 0.17204760015010834, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.18773, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 1.8590359687805176, "learning_rate": 1.2e-05, "loss": 0.2018158733844757, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.22362, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 4.499610424041748, "learning_rate": 1.6000000000000003e-05, "loss": 0.20478995144367218, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.22727, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.5389043092727661, "learning_rate": 2e-05, "loss": 0.1800926923751831, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.19733, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.196098804473877, "learning_rate": 2.4e-05, "loss": 0.15095072984695435, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.16294, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 0.9211762547492981, "learning_rate": 2.8000000000000003e-05, "loss": 0.15947182476520538, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.17289, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.16, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 0.9088688492774963, "learning_rate": 3.2000000000000005e-05, "loss": 0.14260850846767426, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.15328, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.441714882850647, "learning_rate": 3.6e-05, "loss": 0.12682586908340454, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.13522, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 1.6008094549179077, "learning_rate": 4e-05, "loss": 0.11433131992816925, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.12112, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.543449878692627, "learning_rate": 4.4000000000000006e-05, "loss": 0.11057090759277344, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.11692, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 2.326841354370117, "learning_rate": 4.8e-05, "loss": 0.1350790113210678, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.14463, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 1.453883409500122, "learning_rate": 5.2000000000000004e-05, "loss": 0.09902861714363098, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.1041, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 2.738046884536743, "learning_rate": 5.6000000000000006e-05, "loss": 0.1066768541932106, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.11257, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 4.7511372566223145, "learning_rate": 6e-05, "loss": 0.12116716802120209, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.12881, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.0451462268829346, "learning_rate": 6.400000000000001e-05, "loss": 0.08506493270397186, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.08879, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.1784800291061401, "learning_rate": 6.800000000000001e-05, "loss": 0.1078314408659935, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.11386, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.843690276145935, "learning_rate": 7.2e-05, "loss": 0.11107224225997925, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.11748, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.5580672025680542, "learning_rate": 7.6e-05, "loss": 0.12747745215892792, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.13596, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.0499345064163208, "learning_rate": 8e-05, "loss": 0.11054924130439758, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.11689, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.9412247538566589, "learning_rate": 8.4e-05, "loss": 0.1082189604640007, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.11429, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 2.4295451641082764, "learning_rate": 8.800000000000001e-05, "loss": 0.09924432635307312, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.10434, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.8174170255661011, "learning_rate": 9.200000000000001e-05, "loss": 0.11973753571510315, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.1272, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.6467664837837219, "learning_rate": 9.6e-05, "loss": 0.09008632600307465, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.09427, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 4.468479156494141, "learning_rate": 0.0001, "loss": 0.0985565111041069, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.10358, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.3268290758132935, "learning_rate": 9.99990636831587e-05, "loss": 0.11623486876487732, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.12326, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 1.5476082563400269, "learning_rate": 9.999625477159879e-05, "loss": 0.1032816469669342, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.1088, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.869286835193634, "learning_rate": 9.999157338221051e-05, "loss": 0.08588655292987823, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.08968, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.5370637774467468, "learning_rate": 9.998501970980562e-05, "loss": 0.0692746490240097, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.07173, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 1.3889923095703125, "learning_rate": 9.997659402710915e-05, "loss": 0.11194860190153122, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11846, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 3.0530307292938232, "learning_rate": 9.996629668474818e-05, "loss": 0.0718030333518982, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.07444, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 2.0594587326049805, "learning_rate": 9.995412811123711e-05, "loss": 0.07997462153434753, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.08326, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.410278081893921, "learning_rate": 9.994008881295999e-05, "loss": 0.11633460968732834, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.12337, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.787101686000824, "learning_rate": 9.992417937414932e-05, "loss": 0.08981502801179886, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.09397, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.9904115796089172, "learning_rate": 9.99064004568618e-05, "loss": 0.07305086404085159, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.07579, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.848091185092926, "learning_rate": 9.988675280095074e-05, "loss": 0.10084450244903564, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.1061, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.944382905960083, "learning_rate": 9.986523722403528e-05, "loss": 0.07013050466775894, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.07265, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.7226768136024475, "learning_rate": 9.984185462146642e-05, "loss": 0.07395699620246887, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.07676, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.7899583578109741, "learning_rate": 9.98166059662897e-05, "loss": 0.1107742190361023, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.11714, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 1.0888073444366455, "learning_rate": 9.978949230920472e-05, "loss": 0.085332490503788, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.08908, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 1.5716584920883179, "learning_rate": 9.976051477852141e-05, "loss": 0.07090717554092407, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.07348, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.5694183707237244, "learning_rate": 9.972967458011312e-05, "loss": 0.06755677610635757, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.06989, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.44869300723075867, "learning_rate": 9.96969729973664e-05, "loss": 0.06102238595485687, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.06292, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.7561820149421692, "learning_rate": 9.966241139112754e-05, "loss": 0.0809125229716301, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.08428, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.6682188510894775, "learning_rate": 9.96259911996461e-05, "loss": 0.08107280731201172, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.08445, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.63, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.7710041999816895, "learning_rate": 9.958771393851491e-05, "loss": 0.0490361712872982, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.05026, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 1.7858021259307861, "learning_rate": 9.954758120060702e-05, "loss": 0.07058487832546234, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.07314, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 2.4132070541381836, "learning_rate": 9.950559465600948e-05, "loss": 0.04468446969985962, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.0457, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.7, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.7945458292961121, "learning_rate": 9.946175605195379e-05, "loss": 0.01749679446220398, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01765, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 1.3517848253250122, "learning_rate": 9.941606721274322e-05, "loss": 0.05680595710873604, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.05845, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.9590553641319275, "learning_rate": 9.936853003967685e-05, "loss": 0.03352964669466019, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0341, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 3.5370547771453857, "learning_rate": 9.93191465109705e-05, "loss": 0.044728461652994156, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.04574, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.73, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.963900625705719, "learning_rate": 9.926791868167438e-05, "loss": 0.025401750579476357, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.02573, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 1.7401303052902222, "learning_rate": 9.921484868358753e-05, "loss": 0.0454370491206646, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.04649, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 1.119370460510254, "learning_rate": 9.915993872516924e-05, "loss": 0.02946668490767479, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.02991, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.8078593611717224, "learning_rate": 9.9103191091447e-05, "loss": 0.031994231045246124, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.03251, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.9933512806892395, "learning_rate": 9.904460814392147e-05, "loss": 0.04501775652170181, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.04605, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.5195356607437134, "learning_rate": 9.898419232046825e-05, "loss": 0.007048795465379953, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00707, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 20.312498092651367, "learning_rate": 9.892194613523633e-05, "loss": 0.04724446311593056, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.04838, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 1.1221271753311157, "learning_rate": 9.885787217854357e-05, "loss": 0.029883231967687607, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.03033, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.9313758015632629, "learning_rate": 9.879197311676887e-05, "loss": 0.023623952642083168, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.02391, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 1.8007605075836182, "learning_rate": 9.872425169224113e-05, "loss": 0.02848402038216591, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.02889, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.4549485743045807, "learning_rate": 9.865471072312528e-05, "loss": 0.012065699324011803, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01214, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 1.2463338375091553, "learning_rate": 9.858335310330492e-05, "loss": 0.026441413909196854, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.02679, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.7667356133460999, "learning_rate": 9.851018180226185e-05, "loss": 0.02825981006026268, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.02866, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 1.6648930311203003, "learning_rate": 9.843519986495259e-05, "loss": 0.02329423651099205, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.02357, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 1.9009933471679688, "learning_rate": 9.835841041168162e-05, "loss": 0.02009398862719536, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0203, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.8980004191398621, "learning_rate": 9.82798166379715e-05, "loss": 0.01690206490457058, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.01705, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.8615937232971191, "learning_rate": 9.819942181443002e-05, "loss": 0.012583468109369278, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01266, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 1.6823949813842773, "learning_rate": 9.811722928661392e-05, "loss": 0.0463210828602314, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.04741, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 1.1498864889144897, "learning_rate": 9.803324247488975e-05, "loss": 0.03937079757452011, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.04016, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 2.2117602825164795, "learning_rate": 9.794746487429161e-05, "loss": 0.04349299147725105, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.04445, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.9455798268318176, "learning_rate": 9.785990005437554e-05, "loss": 0.010782193392515182, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.01084, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 2.452441930770874, "learning_rate": 9.777055165907117e-05, "loss": 0.04281742870807648, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.04375, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 1.266237735748291, "learning_rate": 9.767942340652993e-05, "loss": 0.01914254203438759, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.01933, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.4602941572666168, "learning_rate": 9.758651908897035e-05, "loss": 0.02264118194580078, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.0229, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.7883008718490601, "learning_rate": 9.749184257252033e-05, "loss": 0.014636470936238766, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01474, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.6581493020057678, "learning_rate": 9.739539779705614e-05, "loss": 0.009349845349788666, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00939, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.24, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.7819415926933289, "learning_rate": 9.729718877603861e-05, "loss": 0.013451536186039448, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.01354, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 1.1575934886932373, "learning_rate": 9.719721959634592e-05, "loss": 0.004510917700827122, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.00452, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 2.9989237785339355, "learning_rate": 9.709549441810375e-05, "loss": 0.05166157707571983, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.05302, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 3.184617757797241, "learning_rate": 9.699201747451195e-05, "loss": 0.04082775115966797, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.04167, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.45952096581459045, "learning_rate": 9.688679307166854e-05, "loss": 0.004115138202905655, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00412, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 1.5055851936340332, "learning_rate": 9.677982558839042e-05, "loss": 0.022924622520804405, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.02319, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 3.981320858001709, "learning_rate": 9.66711194760312e-05, "loss": 0.061497997492551804, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.06343, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 1.282310128211975, "learning_rate": 9.656067925829593e-05, "loss": 0.04182795435190201, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.04272, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.31, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 1.192044734954834, "learning_rate": 9.644850953105288e-05, "loss": 0.02824375033378601, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.02865, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 1.7391445636749268, "learning_rate": 9.633461496214225e-05, "loss": 0.015281446278095245, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.0154, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 3.317592144012451, "learning_rate": 9.621900029118195e-05, "loss": 0.018353678286075592, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01852, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.993459939956665, "learning_rate": 9.610167032937036e-05, "loss": 0.02793055959045887, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.02832, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.8351045846939087, "learning_rate": 9.598262995928611e-05, "loss": 0.03700164332985878, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.03769, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.5809459686279297, "learning_rate": 9.586188413468492e-05, "loss": 0.009170501492917538, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00921, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.7991745471954346, "learning_rate": 9.57394378802934e-05, "loss": 0.057245030999183655, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.05892, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 1.1130009889602661, "learning_rate": 9.56152962916e-05, "loss": 0.030619734898209572, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.03109, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.6239063739776611, "learning_rate": 9.548946453464296e-05, "loss": 0.027802277356386185, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.02819, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.6792464852333069, "learning_rate": 9.53619478457953e-05, "loss": 0.027686573565006256, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.02807, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 1.7035526037216187, "learning_rate": 9.523275153154695e-05, "loss": 0.042529601603746414, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.04345, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.5955518484115601, "learning_rate": 9.51018809682839e-05, "loss": 0.0147638451308012, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.01487, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.9404340386390686, "learning_rate": 9.49693416020645e-05, "loss": 0.01939237490296364, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01958, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 1.0457298755645752, "learning_rate": 9.483513894839276e-05, "loss": 0.025921236723661423, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.02626, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.6194956302642822, "learning_rate": 9.469927859198888e-05, "loss": 0.008739783428609371, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00878, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.08649761974811554, "learning_rate": 9.456176618655689e-05, "loss": 0.0009364818106405437, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00094, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.22556327283382416, "learning_rate": 9.442260745454927e-05, "loss": 0.0024343321565538645, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.00244, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 1.0753309726715088, "learning_rate": 9.428180818692884e-05, "loss": 0.013819430023431778, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.01392, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 1.1136362552642822, "learning_rate": 9.413937424292791e-05, "loss": 0.025022894144058228, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.02534, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 5.471511363983154, "learning_rate": 9.399531154980424e-05, "loss": 0.009748886339366436, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.0098, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 2.097762107849121, "learning_rate": 9.384962610259455e-05, "loss": 0.02835565060377121, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.02876, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 2.5249078273773193, "learning_rate": 9.370232396386494e-05, "loss": 0.03050174005329609, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.03097, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 1.2749090194702148, "learning_rate": 9.355341126345868e-05, "loss": 0.03686698526144028, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.03756, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 1.0384973287582397, "learning_rate": 9.340289419824107e-05, "loss": 0.010501891374588013, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.01056, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.7338583469390869, "learning_rate": 9.325077903184159e-05, "loss": 0.015517533756792545, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.01564, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 1.508381724357605, "learning_rate": 9.30970720943932e-05, "loss": 0.026122786104679108, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02647, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 1.5016816854476929, "learning_rate": 9.2941779782269e-05, "loss": 0.012254172936081886, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.01233, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.6668992638587952, "learning_rate": 9.278490855781596e-05, "loss": 0.01190841943025589, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01198, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 1.4711428880691528, "learning_rate": 9.262646494908604e-05, "loss": 0.013266445137560368, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.01335, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.23678848147392273, "learning_rate": 9.246645554956457e-05, "loss": 0.0059791747480630875, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.006, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.9577431082725525, "learning_rate": 9.230488701789578e-05, "loss": 0.0124357920140028, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.01251, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.8213187456130981, "learning_rate": 9.214176607760577e-05, "loss": 0.017692942172288895, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.01785, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.39163705706596375, "learning_rate": 9.197709951682268e-05, "loss": 0.013661626726388931, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01376, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.7322210073471069, "learning_rate": 9.181089418799428e-05, "loss": 0.007864446379244328, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.0079, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.11670327186584473, "learning_rate": 9.164315700760271e-05, "loss": 0.002284077927470207, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00229, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.42723026871681213, "learning_rate": 9.147389495587671e-05, "loss": 0.006347275339066982, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.00637, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.16643336415290833, "learning_rate": 9.130311507650116e-05, "loss": 0.0030799799133092165, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00308, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 4.5798163414001465, "learning_rate": 9.113082447632394e-05, "loss": 0.02847391366958618, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.02888, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.6771937608718872, "learning_rate": 9.09570303250602e-05, "loss": 0.006078810431063175, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0061, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.249184712767601, "learning_rate": 9.078173985499394e-05, "loss": 0.00254970695823431, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00255, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.07522158324718475, "learning_rate": 9.060496036067713e-05, "loss": 0.0011659915326163173, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00117, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.8817594647407532, "learning_rate": 9.042669919862615e-05, "loss": 0.004834473133087158, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00485, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.315163254737854, "learning_rate": 9.024696378701557e-05, "loss": 0.003778901882469654, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00379, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.6788947582244873, "learning_rate": 9.006576160536948e-05, "loss": 0.004974414594471455, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00499, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 1.1690468788146973, "learning_rate": 8.988310019425035e-05, "loss": 0.01107865385711193, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01114, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.6978726983070374, "learning_rate": 8.969898715494506e-05, "loss": 0.01649560034275055, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01663, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 1.1985689401626587, "learning_rate": 8.951343014914869e-05, "loss": 0.01503828540444374, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01515, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.0969144105911255, "learning_rate": 8.932643689864568e-05, "loss": 0.019518010318279266, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01971, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 1.6939164400100708, "learning_rate": 8.913801518498845e-05, "loss": 0.030688317492604256, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03116, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.7046255469322205, "learning_rate": 8.894817284917364e-05, "loss": 0.01922966167330742, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.01942, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.5283721685409546, "learning_rate": 8.875691779131569e-05, "loss": 0.010968795977532864, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.01103, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.28, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 0.8876633048057556, "learning_rate": 8.856425797031829e-05, "loss": 0.03537715971469879, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.03601, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 0.7710462808609009, "learning_rate": 8.837020140354295e-05, "loss": 0.03557523339986801, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.03622, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.4544171392917633, "learning_rate": 8.817475616647554e-05, "loss": 0.011630062013864517, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0117, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.45951512455940247, "learning_rate": 8.797793039239017e-05, "loss": 0.0075307684019207954, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00756, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.5971090793609619, "learning_rate": 8.777973227201069e-05, "loss": 0.012561138719320297, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.01264, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.27056244015693665, "learning_rate": 8.758017005316988e-05, "loss": 0.004323258996009827, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00433, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.12034580111503601, "learning_rate": 8.737925204046629e-05, "loss": 0.0026907543651759624, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00269, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 1.4877914190292358, "learning_rate": 8.717698659491851e-05, "loss": 0.033646970987319946, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.03422, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 1.0115147829055786, "learning_rate": 8.697338213361735e-05, "loss": 0.01678440161049366, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01693, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.3534937798976898, "learning_rate": 8.676844712937552e-05, "loss": 0.011359256692230701, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.01142, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.7316568493843079, "learning_rate": 8.656219011037509e-05, "loss": 0.01793782413005829, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.0181, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.8656009435653687, "learning_rate": 8.63546196598125e-05, "loss": 0.012522783130407333, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0126, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.8543619513511658, "learning_rate": 8.614574441554145e-05, "loss": 0.02325616031885147, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.02353, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.5315729379653931, "learning_rate": 8.593557306971349e-05, "loss": 0.011839472688734531, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.01191, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.640302300453186, "learning_rate": 8.572411436841618e-05, "loss": 0.005968144163489342, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00599, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.691736102104187, "learning_rate": 8.551137711130922e-05, "loss": 0.022184286266565323, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.02243, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.3409017324447632, "learning_rate": 8.529737015125824e-05, "loss": 0.007027980871498585, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00705, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.7031590938568115, "learning_rate": 8.508210239396639e-05, "loss": 0.010952194221317768, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.01101, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.1663326472043991, "learning_rate": 8.486558279760375e-05, "loss": 0.001918938709422946, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00192, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.26013869047164917, "learning_rate": 8.464782037243449e-05, "loss": 0.0038864361122250557, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00389, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.6548788547515869, "learning_rate": 8.442882418044202e-05, "loss": 0.005758331622928381, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00577, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.4741109609603882, "learning_rate": 8.420860333495179e-05, "loss": 0.008476601913571358, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00851, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.48784565925598145, "learning_rate": 8.398716700025208e-05, "loss": 0.007597742602229118, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00763, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.7200278043746948, "learning_rate": 8.376452439121266e-05, "loss": 0.026581021025776863, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.02694, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.7970268130302429, "learning_rate": 8.354068477290124e-05, "loss": 0.01903615891933441, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.01922, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.8663705587387085, "learning_rate": 8.331565746019807e-05, "loss": 0.014783211052417755, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01489, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.8002447485923767, "learning_rate": 8.308945181740812e-05, "loss": 0.006486333906650543, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00651, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.9022852778434753, "learning_rate": 8.286207725787153e-05, "loss": 0.01912708580493927, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.01931, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 1.0369495153427124, "learning_rate": 8.263354324357182e-05, "loss": 0.032328952103853226, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.03286, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.7037595510482788, "learning_rate": 8.240385928474219e-05, "loss": 0.012159975245594978, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01223, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.27457350492477417, "learning_rate": 8.217303493946967e-05, "loss": 0.006982964463531971, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00701, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.14809690415859222, "learning_rate": 8.194107981329746e-05, "loss": 0.0033493328373879194, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00335, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.807507336139679, "learning_rate": 8.170800355882518e-05, "loss": 0.027879539877176285, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.02827, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.8214336037635803, "learning_rate": 8.147381587530713e-05, "loss": 0.011526726186275482, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.01159, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.6428171396255493, "learning_rate": 8.123852650824877e-05, "loss": 0.012080689892172813, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.01215, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.2013300657272339, "learning_rate": 8.100214524900103e-05, "loss": 0.005223957821726799, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00524, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.47269052267074585, "learning_rate": 8.076468193435301e-05, "loss": 0.015599744394421577, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.01572, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.6179898977279663, "learning_rate": 8.052614644612253e-05, "loss": 0.01663314364850521, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01677, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.4822327196598053, "learning_rate": 8.028654871074489e-05, "loss": 0.016431357711553574, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01657, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.32778412103652954, "learning_rate": 8.004589869885986e-05, "loss": 0.007131228223443031, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00716, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.38234439492225647, "learning_rate": 7.980420642489674e-05, "loss": 0.005195335950702429, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00521, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 34.03, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.6139643788337708, "learning_rate": 7.95614819466576e-05, "loss": 0.017041798681020737, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01719, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.4943687617778778, "learning_rate": 7.931773536489872e-05, "loss": 0.010258497670292854, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01031, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.48631641268730164, "learning_rate": 7.907297682291035e-05, "loss": 0.013852249830961227, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01395, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.2863270342350006, "learning_rate": 7.882721650609442e-05, "loss": 0.004168541170656681, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.00418, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.42346036434173584, "learning_rate": 7.85804646415409e-05, "loss": 0.008716818876564503, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00875, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.2628840506076813, "learning_rate": 7.833273149760207e-05, "loss": 0.007842686027288437, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00787, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.9532155990600586, "learning_rate": 7.808402738346527e-05, "loss": 0.01150444895029068, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.01157, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 1.2822916507720947, "learning_rate": 7.783436264872382e-05, "loss": 0.03626871854066849, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.03693, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.3439704179763794, "learning_rate": 7.758374768294647e-05, "loss": 0.004018259700387716, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00403, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.33922433853149414, "learning_rate": 7.733219291524489e-05, "loss": 0.006022469140589237, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00604, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.13645493984222412, "learning_rate": 7.707970881383977e-05, "loss": 0.001670514466241002, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.00167, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.37, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.6642374992370605, "learning_rate": 7.682630588562518e-05, "loss": 0.009863261133432388, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00991, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 1.5430012941360474, "learning_rate": 7.657199467573129e-05, "loss": 0.006985161919146776, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00701, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.5563443303108215, "learning_rate": 7.631678576708561e-05, "loss": 0.019615978002548218, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01981, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.6395878195762634, "learning_rate": 7.606068977997255e-05, "loss": 0.007719866000115871, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00775, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.13239151239395142, "learning_rate": 7.580371737159148e-05, "loss": 0.00217211595736444, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.00217, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 1.0727325677871704, "learning_rate": 7.554587923561324e-05, "loss": 0.006420600693672895, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00644, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.735874593257904, "learning_rate": 7.528718610173511e-05, "loss": 0.013715693727135658, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01381, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.30030176043510437, "learning_rate": 7.502764873523431e-05, "loss": 0.0014124787412583828, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00141, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.48357027769088745, "learning_rate": 7.476727793652011e-05, "loss": 0.005036189220845699, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00505, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.05488593131303787, "learning_rate": 7.450608454068415e-05, "loss": 0.001089503988623619, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00109, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.08703088760375977, "learning_rate": 7.424407941704987e-05, "loss": 0.0017282667104154825, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00173, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.6883617639541626, "learning_rate": 7.398127346871986e-05, "loss": 0.024259766563773155, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.02456, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.0922660306096077, "learning_rate": 7.371767763212238e-05, "loss": 0.0017141795251518488, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00172, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.0371268093585968, "learning_rate": 7.345330287655617e-05, "loss": 0.0008292457787320018, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00083, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.5611706376075745, "learning_rate": 7.31881602037339e-05, "loss": 0.018400216475129128, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01857, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.6445966362953186, "learning_rate": 7.29222606473245e-05, "loss": 0.010754700750112534, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01081, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.12240605056285858, "learning_rate": 7.265561527249383e-05, "loss": 0.0029469646979123354, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00295, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.2460574358701706, "learning_rate": 7.238823517544436e-05, "loss": 0.005135843064635992, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00515, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.15310531854629517, "learning_rate": 7.212013148295333e-05, "loss": 0.001555054564960301, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00156, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.22622451186180115, "learning_rate": 7.185131535190975e-05, "loss": 0.006894643418490887, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00692, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.06656525284051895, "learning_rate": 7.158179796885005e-05, "loss": 0.0013012258568778634, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0013, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.6946818828582764, "learning_rate": 7.131159054949273e-05, "loss": 0.018886703997850418, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.01907, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.24958696961402893, "learning_rate": 7.104070433827139e-05, "loss": 0.0034609180875122547, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00347, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.15482349693775177, "learning_rate": 7.076915060786705e-05, "loss": 0.0038165219593793154, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00382, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.18793755769729614, "learning_rate": 7.049694065873882e-05, "loss": 0.0029448773711919785, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.00295, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.2758285701274872, "learning_rate": 7.022408581865382e-05, "loss": 0.004630815237760544, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00464, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.1864178329706192, "learning_rate": 6.99505974422157e-05, "loss": 0.002394351176917553, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.0024, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.35, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.06263653934001923, "learning_rate": 6.967648691039213e-05, "loss": 0.0006641596555709839, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00066, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.060556262731552124, "learning_rate": 6.940176563004123e-05, "loss": 0.0009560499456711113, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00096, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.7326021790504456, "learning_rate": 6.912644503343682e-05, "loss": 0.007779562380164862, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00781, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.21277542412281036, "learning_rate": 6.885053657779273e-05, "loss": 0.0024171732366085052, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00242, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 21.03, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.1825384646654129, "learning_rate": 6.857405174478604e-05, "loss": 0.0012138721067458391, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00121, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.30992576479911804, "learning_rate": 6.82970020400792e-05, "loss": 0.012609429657459259, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01269, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.10836701840162277, "learning_rate": 6.801939899284132e-05, "loss": 0.0009367704624310136, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.00094, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 30.14, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.5945758819580078, "learning_rate": 6.774125415526827e-05, "loss": 0.014650771394371986, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.01476, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.35, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.2690686583518982, "learning_rate": 6.746257910210214e-05, "loss": 0.002037809696048498, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00204, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.012568900361657143, "learning_rate": 6.718338543014937e-05, "loss": 0.0002361374208703637, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00024, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 33.91, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.11370351910591125, "learning_rate": 6.69036847577983e-05, "loss": 0.0005663428455591202, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.00057, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.3703382611274719, "learning_rate": 6.662348872453553e-05, "loss": 0.001750132068991661, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00175, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.1673813760280609, "learning_rate": 6.63428089904618e-05, "loss": 0.001005021738819778, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00101, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.5673899054527283, "learning_rate": 6.60616572358065e-05, "loss": 0.004386983346194029, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.0044, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.5597986578941345, "learning_rate": 6.578004516044172e-05, "loss": 0.006121012382209301, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.00614, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.0772668793797493, "learning_rate": 6.549798448339548e-05, "loss": 9.52573464019224e-05, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.0001, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.36613792181015015, "learning_rate": 6.521548694236384e-05, "loss": 0.0053746518678963184, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00539, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.01294800080358982, "learning_rate": 6.493256429322259e-05, "loss": 0.00023405796673614532, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00023, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.7735905051231384, "learning_rate": 6.464922830953799e-05, "loss": 0.004496313165873289, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00451, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.24556618928909302, "learning_rate": 6.436549078207688e-05, "loss": 0.0014010637532919645, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0014, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.006141041871160269, "learning_rate": 6.408136351831592e-05, "loss": 9.984963980969042e-05, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.0001, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.8094276785850525, "learning_rate": 6.379685834195036e-05, "loss": 0.016288109123706818, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01642, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.5283324122428894, "learning_rate": 6.351198709240186e-05, "loss": 0.00441674143075943, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00443, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.7631818652153015, "learning_rate": 6.32267616243259e-05, "loss": 0.01268005184829235, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01276, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.6164374947547913, "learning_rate": 6.294119380711849e-05, "loss": 0.007528468035161495, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00756, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.04845655336976051, "learning_rate": 6.265529552442209e-05, "loss": 0.0008347132243216038, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00084, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.7650155425071716, "learning_rate": 6.236907867363127e-05, "loss": 0.013833905570209026, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01393, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.08520758152008057, "learning_rate": 6.208255516539749e-05, "loss": 0.0013073883019387722, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00131, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.060544177889823914, "learning_rate": 6.179573692313344e-05, "loss": 0.0011635713744908571, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00116, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.05938483774662018, "learning_rate": 6.150863588251694e-05, "loss": 0.0006758210365660489, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00068, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.31434205174446106, "learning_rate": 6.122126399099419e-05, "loss": 0.0025327191688120365, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00254, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.34857723116874695, "learning_rate": 6.0933633207282615e-05, "loss": 0.005136602092534304, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00515, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.6858780384063721, "learning_rate": 6.064575550087316e-05, "loss": 0.014921127818524837, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.01503, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.5875364542007446, "learning_rate": 6.0357642851532245e-05, "loss": 0.013166015036404133, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.01325, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.51, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.016892392188310623, "learning_rate": 6.0069307248803294e-05, "loss": 0.00013251579366624355, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00013, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.02904786355793476, "learning_rate": 5.9780760691507635e-05, "loss": 0.0004572165198624134, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00046, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.21986009180545807, "learning_rate": 5.9492015187245334e-05, "loss": 0.00314087001606822, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00315, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.14051926136016846, "learning_rate": 5.920308275189541e-05, "loss": 0.001373404636979103, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.00137, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.04658694937825203, "learning_rate": 5.8913975409115874e-05, "loss": 0.0007025808445177972, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0007, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.050223592668771744, "learning_rate": 5.8624705189843395e-05, "loss": 9.595624578651041e-05, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0001, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 30.14, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.02413795329630375, "learning_rate": 5.833528413179249e-05, "loss": 0.0003691620077006519, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00037, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.280560165643692, "learning_rate": 5.80457242789548e-05, "loss": 0.004048904404044151, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00406, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.0882704108953476, "learning_rate": 5.77560376810977e-05, "loss": 0.0015867534093558788, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00159, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.283220499753952, "learning_rate": 5.7466236393263005e-05, "loss": 0.0025843377225100994, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00259, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.03943847119808197, "learning_rate": 5.717633247526522e-05, "loss": 0.0005022862460464239, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.0005, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.07193606346845627, "learning_rate": 5.688633799118971e-05, "loss": 0.00030229712137952447, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.0003, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.2413291335105896, "learning_rate": 5.659626500889066e-05, "loss": 0.002245324896648526, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00225, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 30.92, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.021189754828810692, "learning_rate": 5.6306125599488905e-05, "loss": 0.00015861231076996773, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.00016, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.3592158257961273, "learning_rate": 5.601593183686955e-05, "loss": 0.0035024790558964014, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.00351, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.004493708722293377, "learning_rate": 5.572569579717961e-05, "loss": 8.90940209501423e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00009, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.03871360048651695, "learning_rate": 5.543542955832538e-05, "loss": 0.0003978379536420107, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.0004, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.4253173768520355, "learning_rate": 5.514514519946986e-05, "loss": 0.004136234056204557, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00414, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.05398097261786461, "learning_rate": 5.485485480053015e-05, "loss": 0.0007250997587107122, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00073, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.7172045111656189, "learning_rate": 5.4564570441674645e-05, "loss": 0.006868102587759495, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00689, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.15535487234592438, "learning_rate": 5.42743042028204e-05, "loss": 0.0012845955789089203, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00129, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.20726194977760315, "learning_rate": 5.3984068163130464e-05, "loss": 0.0029339673928916454, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00294, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.7013916969299316, "learning_rate": 5.369387440051111e-05, "loss": 0.0007347581558860838, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00074, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.3672807812690735, "learning_rate": 5.340373499110935e-05, "loss": 0.005670428276062012, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00569, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.03397063910961151, "learning_rate": 5.3113662008810304e-05, "loss": 0.0003728395968209952, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00037, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.026753786951303482, "learning_rate": 5.282366752473479e-05, "loss": 0.00026775297010317445, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00027, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.14527013897895813, "learning_rate": 5.2533763606737005e-05, "loss": 0.001945782103575766, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00195, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.9700937271118164, "learning_rate": 5.224396231890232e-05, "loss": 0.011370252817869186, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01144, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.07820557057857513, "learning_rate": 5.195427572104522e-05, "loss": 0.0006644683890044689, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00066, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.03898119553923607, "learning_rate": 5.166471586820751e-05, "loss": 0.0003087117802351713, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00031, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.020814185962080956, "learning_rate": 5.1375294810156615e-05, "loss": 0.00018373504281044006, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00018, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.0038712858222424984, "learning_rate": 5.1086024590884144e-05, "loss": 5.5644792155362666e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00006, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.007198450155556202, "learning_rate": 5.079691724810461e-05, "loss": 0.00010536171612329781, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00011, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.02198565937578678, "learning_rate": 5.0507984812754684e-05, "loss": 0.0002986161853186786, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0003, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.15455780923366547, "learning_rate": 5.021923930849237e-05, "loss": 0.0014275156427174807, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00143, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.040049366652965546, "learning_rate": 4.99306927511967e-05, "loss": 0.0006070019444450736, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00061, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.009265661239624023, "learning_rate": 4.964235714846775e-05, "loss": 0.00016158097423613071, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00016, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.9242985248565674, "learning_rate": 4.9354244499126866e-05, "loss": 0.003331332467496395, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00334, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.8444964289665222, "learning_rate": 4.90663667927174e-05, "loss": 0.010772204957902431, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01083, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 1.2778130769729614, "learning_rate": 4.877873600900581e-05, "loss": 0.0037558721378445625, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00376, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.002278241328895092, "learning_rate": 4.849136411748306e-05, "loss": 3.959356399718672e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00004, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.02480052225291729, "learning_rate": 4.8204263076866574e-05, "loss": 0.0002666396030690521, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00027, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.004437324125319719, "learning_rate": 4.791744483460251e-05, "loss": 7.775815174682066e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00008, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.07193279266357422, "learning_rate": 4.7630921326368736e-05, "loss": 0.0008455555653199553, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00085, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 20.06, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 2.9605820178985596, "learning_rate": 4.7344704475577916e-05, "loss": 0.0020671829115599394, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.00207, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 30.91, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.11382972449064255, "learning_rate": 4.705880619288153e-05, "loss": 0.0015217619948089123, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00152, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.14022445678710938, "learning_rate": 4.677323837567412e-05, "loss": 0.0016303453594446182, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00163, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.04894035682082176, "learning_rate": 4.6488012907598146e-05, "loss": 0.0005147549672983587, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00051, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.23412460088729858, "learning_rate": 4.620314165804964e-05, "loss": 0.004053084179759026, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00406, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.00910744909197092, "learning_rate": 4.591863648168407e-05, "loss": 0.00012951975804753602, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00013, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.07301023602485657, "learning_rate": 4.5634509217923135e-05, "loss": 0.001006174017675221, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00101, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 31.25, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.17979340255260468, "learning_rate": 4.535077169046201e-05, "loss": 0.0019607101567089558, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.00196, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.17852960526943207, "learning_rate": 4.506743570677743e-05, "loss": 0.003147775772958994, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00315, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.017762113362550735, "learning_rate": 4.478451305763618e-05, "loss": 0.00011041737161576748, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00011, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.038158636540174484, "learning_rate": 4.450201551660454e-05, "loss": 0.0002535527746658772, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00025, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.15363644063472748, "learning_rate": 4.4219954839558276e-05, "loss": 0.0013960065552964807, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.0014, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.53, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.008249739184975624, "learning_rate": 4.393834276419352e-05, "loss": 7.491804717574269e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00007, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.07293487340211868, "learning_rate": 4.36571910095382e-05, "loss": 0.00022738361440133303, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00023, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.015550420619547367, "learning_rate": 4.337651127546448e-05, "loss": 0.00015006406465545297, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00015, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.010072221048176289, "learning_rate": 4.3096315242201736e-05, "loss": 9.079546725843102e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00009, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 31.25, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.14179487526416779, "learning_rate": 4.2816614569850635e-05, "loss": 0.0009526675567030907, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.00095, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.004218440968543291, "learning_rate": 4.2537420897897864e-05, "loss": 5.490595503943041e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00005, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.39, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.03395330533385277, "learning_rate": 4.225874584473174e-05, "loss": 0.00019801879534497857, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0002, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.02219381555914879, "learning_rate": 4.19806010071587e-05, "loss": 0.0001903845986817032, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00019, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.30774810910224915, "learning_rate": 4.170299795992081e-05, "loss": 0.0016865580109879375, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00169, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.014054778032004833, "learning_rate": 4.142594825521398e-05, "loss": 0.00012074395635863766, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00012, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.0012107209768146276, "learning_rate": 4.114946342220728e-05, "loss": 1.623263415240217e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00002, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.005954182241111994, "learning_rate": 4.087355496656321e-05, "loss": 4.829532190342434e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00005, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.0032241414301097393, "learning_rate": 4.05982343699588e-05, "loss": 3.17503945552744e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00003, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.00588262639939785, "learning_rate": 4.0323513089607876e-05, "loss": 7.485682726837695e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00007, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 1.1177337169647217, "learning_rate": 4.004940255778431e-05, "loss": 0.006013314239680767, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00603, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.003970997408032417, "learning_rate": 3.977591418134619e-05, "loss": 3.211823059245944e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00003, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.8645995259284973, "learning_rate": 3.95030593412612e-05, "loss": 0.011767236515879631, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01184, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.5, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.4608267843723297, "learning_rate": 3.923084939213296e-05, "loss": 0.003401111578568816, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00341, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.0016024999786168337, "learning_rate": 3.895929566172861e-05, "loss": 2.3912249162094668e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00002, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.0071640582755208015, "learning_rate": 3.868840945050728e-05, "loss": 6.729915912728757e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00007, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.3901693820953369, "learning_rate": 3.841820203114995e-05, "loss": 0.002705880207940936, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.00271, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.3634466528892517, "learning_rate": 3.814868464809027e-05, "loss": 0.0023742138873785734, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00238, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.3598405122756958, "learning_rate": 3.787986851704667e-05, "loss": 0.0030554872937500477, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00306, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.15283675491809845, "learning_rate": 3.7611764824555654e-05, "loss": 0.001057734014466405, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00106, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 1.113755464553833, "learning_rate": 3.734438472750619e-05, "loss": 0.0056664347648620605, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00568, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.21582241356372833, "learning_rate": 3.707773935267552e-05, "loss": 0.002649337984621525, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00265, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.11977498978376389, "learning_rate": 3.68118397962661e-05, "loss": 0.0006798164686188102, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00068, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.8377370834350586, "learning_rate": 3.654669712344384e-05, "loss": 0.017718864604830742, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.01788, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.71, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.4623136818408966, "learning_rate": 3.628232236787763e-05, "loss": 0.016269022598862648, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0164, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.3409557640552521, "learning_rate": 3.6018726531280144e-05, "loss": 0.008879177272319794, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.00892, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.6244899034500122, "learning_rate": 3.575592058295017e-05, "loss": 0.01244965847581625, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01253, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.7060756683349609, "learning_rate": 3.549391545931585e-05, "loss": 0.00534452311694622, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00536, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.3109199106693268, "learning_rate": 3.5232722063479914e-05, "loss": 0.0035857076290994883, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00359, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.252116858959198, "learning_rate": 3.49723512647657e-05, "loss": 0.0019834409467875957, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00199, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.25215592980384827, "learning_rate": 3.471281389826491e-05, "loss": 0.0027561055030673742, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00276, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.25582870841026306, "learning_rate": 3.4454120764386764e-05, "loss": 0.0019186809659004211, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00192, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.25820738077163696, "learning_rate": 3.4196282628408526e-05, "loss": 0.003771770978346467, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00378, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.4859471619129181, "learning_rate": 3.3939310220027456e-05, "loss": 0.003374667838215828, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00338, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.82, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.03647678345441818, "learning_rate": 3.3683214232914404e-05, "loss": 0.0005341343930922449, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00053, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.13029053807258606, "learning_rate": 3.342800532426873e-05, "loss": 0.0013402426848188043, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00134, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.007070130668580532, "learning_rate": 3.317369411437484e-05, "loss": 7.39723545848392e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00007, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.04079952463507652, "learning_rate": 3.292029118616024e-05, "loss": 0.0005641378229483962, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00056, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.0809498280286789, "learning_rate": 3.266780708475511e-05, "loss": 0.0006574043072760105, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00066, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.03199659660458565, "learning_rate": 3.241625231705354e-05, "loss": 0.00026542891282588243, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00027, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 2.9403843879699707, "learning_rate": 3.216563735127618e-05, "loss": 0.012433270923793316, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.01251, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.24480563402175903, "learning_rate": 3.191597261653475e-05, "loss": 0.003892891574651003, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0039, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.30624324083328247, "learning_rate": 3.166726850239794e-05, "loss": 0.0036293610464781523, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00364, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.043700847774744034, "learning_rate": 3.141953535845912e-05, "loss": 0.00042901598499156535, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00043, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.61, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 1.8790591955184937, "learning_rate": 3.11727834939056e-05, "loss": 0.002974341157823801, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00298, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.29022395610809326, "learning_rate": 3.092702317708967e-05, "loss": 0.005272896960377693, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00529, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.42792463302612305, "learning_rate": 3.0682264635101276e-05, "loss": 0.0023785533849149942, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.00238, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.018285734578967094, "learning_rate": 3.0438518053342407e-05, "loss": 0.00024944631149992347, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00025, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.038543861359357834, "learning_rate": 3.0195793575103266e-05, "loss": 0.0007904964732006192, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00079, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.32, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.8656831979751587, "learning_rate": 2.9954101301140146e-05, "loss": 0.005089838523417711, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.0051, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.13004359602928162, "learning_rate": 2.9713451289255123e-05, "loss": 0.001057770336046815, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00106, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.13675245642662048, "learning_rate": 2.9473853553877484e-05, "loss": 0.0016169004375115037, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00162, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.008903692476451397, "learning_rate": 2.9235318065647e-05, "loss": 0.00013879280595574528, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00014, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.5208345055580139, "learning_rate": 2.8997854750998964e-05, "loss": 0.010207434184849262, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.01026, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.01949097216129303, "learning_rate": 2.8761473491751258e-05, "loss": 0.0002005547285079956, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.0002, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.22553233802318573, "learning_rate": 2.8526184124692883e-05, "loss": 0.001286161132156849, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00129, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.6486837863922119, "learning_rate": 2.829199644117484e-05, "loss": 0.006423979997634888, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00644, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.1089920923113823, "learning_rate": 2.8058920186702553e-05, "loss": 0.0007982183597050607, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0008, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.3139247000217438, "learning_rate": 2.782696506053033e-05, "loss": 0.0027510258369147778, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00275, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.2806653678417206, "learning_rate": 2.7596140715257824e-05, "loss": 0.0004386453947518021, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00044, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 22.04, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.019236773252487183, "learning_rate": 2.7366456756428184e-05, "loss": 0.0003082782495766878, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00031, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.00950646959245205, "learning_rate": 2.7137922742128486e-05, "loss": 0.00012520923337433487, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00013, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.4488280415534973, "learning_rate": 2.691054818259188e-05, "loss": 0.011222360655665398, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.01129, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.21593934297561646, "learning_rate": 2.6684342539801933e-05, "loss": 0.0021973371040076017, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0022, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.005680008791387081, "learning_rate": 2.645931522709877e-05, "loss": 9.545594366500154e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.0001, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.59, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.13788048923015594, "learning_rate": 2.6235475608787365e-05, "loss": 0.0005434236954897642, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00054, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.2276841700077057, "learning_rate": 2.6012832999747916e-05, "loss": 0.003898271592333913, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00391, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.42465123534202576, "learning_rate": 2.579139666504821e-05, "loss": 0.007033151108771563, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00706, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.09408481419086456, "learning_rate": 2.557117581955798e-05, "loss": 0.0010467548854649067, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00105, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.2558613717556, "learning_rate": 2.5352179627565532e-05, "loss": 0.0037579338531941175, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00377, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.03390956297516823, "learning_rate": 2.5134417202396277e-05, "loss": 0.00047905120300129056, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00048, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.013432999141514301, "learning_rate": 2.491789760603361e-05, "loss": 0.0002543188747949898, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00025, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.21234621107578278, "learning_rate": 2.4702629848741764e-05, "loss": 0.0025126810651272535, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00252, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.03733818978071213, "learning_rate": 2.4488622888690785e-05, "loss": 0.0004897401086054742, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00049, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.04186299443244934, "learning_rate": 2.427588563158384e-05, "loss": 0.0003400795394554734, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00034, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.020194999873638153, "learning_rate": 2.406442693028651e-05, "loss": 0.0004338558064773679, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00043, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.03726119548082352, "learning_rate": 2.3854255584458547e-05, "loss": 0.0003977447922807187, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0004, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.08788038790225983, "learning_rate": 2.3645380340187508e-05, "loss": 0.0009859498823061585, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00099, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.007118840701878071, "learning_rate": 2.3437809889624914e-05, "loss": 0.00010354513506172225, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.0001, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.008573561906814575, "learning_rate": 2.3231552870624487e-05, "loss": 0.0001520750083727762, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00015, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.1414867788553238, "learning_rate": 2.3026617866382657e-05, "loss": 0.001000015065073967, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.001, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.25723764300346375, "learning_rate": 2.2823013405081507e-05, "loss": 0.0022035606671124697, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00221, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.5197186470031738, "learning_rate": 2.2620747959533722e-05, "loss": 0.013849914073944092, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01395, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.4582887291908264, "learning_rate": 2.2419829946830123e-05, "loss": 0.004607388284057379, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00462, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.14644519984722137, "learning_rate": 2.2220267727989325e-05, "loss": 0.0016854527639225125, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.00169, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.5693517923355103, "learning_rate": 2.202206960760984e-05, "loss": 0.008553116582334042, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.00859, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.029738899320364, "learning_rate": 2.182524383352446e-05, "loss": 0.0004957930650562048, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0005, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.32796767354011536, "learning_rate": 2.1629798596457056e-05, "loss": 0.002648593857884407, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00265, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.07436935603618622, "learning_rate": 2.1435742029681725e-05, "loss": 0.0007996325148269534, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.0008, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.11638450622558594, "learning_rate": 2.124308220868431e-05, "loss": 0.0013446620432659984, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00135, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.013861642219126225, "learning_rate": 2.105182715082638e-05, "loss": 0.00016913353465497494, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00017, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.07333674281835556, "learning_rate": 2.0861984815011552e-05, "loss": 0.00042261957423761487, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00042, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.43776002526283264, "learning_rate": 2.0673563101354323e-05, "loss": 0.002082346472889185, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00208, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.26594752073287964, "learning_rate": 2.0486569850851317e-05, "loss": 0.00419085705652833, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0042, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.012375804595649242, "learning_rate": 2.0301012845054956e-05, "loss": 0.00013265288725961, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00013, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.026575617492198944, "learning_rate": 2.011689980574966e-05, "loss": 0.00030461009009741247, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0003, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.2626779079437256, "learning_rate": 1.993423839463052e-05, "loss": 0.0023803627118468285, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00238, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.34402862191200256, "learning_rate": 1.975303621298445e-05, "loss": 0.007769954390823841, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.0078, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.08672498911619186, "learning_rate": 1.957330080137385e-05, "loss": 0.000926541571971029, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00093, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.033894505351781845, "learning_rate": 1.9395039639322864e-05, "loss": 0.00036656440352089703, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00037, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.033202048391103745, "learning_rate": 1.9218260145006073e-05, "loss": 9.327141742687672e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00009, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.0983816608786583, "learning_rate": 1.904296967493982e-05, "loss": 0.002152298344299197, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00215, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.08792620897293091, "learning_rate": 1.8869175523676064e-05, "loss": 0.0007333102985285223, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00073, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.048296112567186356, "learning_rate": 1.869688492349885e-05, "loss": 0.0004775348352268338, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00048, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.07831458747386932, "learning_rate": 1.85261050441233e-05, "loss": 0.0007663497817702591, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00077, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.12764236330986023, "learning_rate": 1.8356842992397304e-05, "loss": 0.0014378696214407682, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00144, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.0030040168203413486, "learning_rate": 1.8189105812005714e-05, "loss": 1.8676728359423578e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00002, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.01588471233844757, "learning_rate": 1.802290048317732e-05, "loss": 0.00018909858772531152, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00019, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.07266035676002502, "learning_rate": 1.785823392239424e-05, "loss": 0.0007789736846461892, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00078, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.05916104093194008, "learning_rate": 1.7695112982104225e-05, "loss": 0.00025843404000625014, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00026, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.0031170935835689306, "learning_rate": 1.7533544450435433e-05, "loss": 2.8304084480623715e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00003, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.33631381392478943, "learning_rate": 1.7373535050913946e-05, "loss": 0.0030236607417464256, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00303, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.0025229603052139282, "learning_rate": 1.721509144218405e-05, "loss": 4.416225783643313e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00004, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.11602852493524551, "learning_rate": 1.705822021773101e-05, "loss": 0.0005808707792311907, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00058, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.0960102379322052, "learning_rate": 1.69029279056068e-05, "loss": 0.00027728226268664, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00028, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.027595622465014458, "learning_rate": 1.6749220968158415e-05, "loss": 0.00028878013836219907, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00029, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.007488140370696783, "learning_rate": 1.659710580175893e-05, "loss": 9.729081648401916e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.0001, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.21081773936748505, "learning_rate": 1.644658873654133e-05, "loss": 0.003025627229362726, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00303, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.0414726659655571, "learning_rate": 1.629767603613508e-05, "loss": 0.00038525485433638096, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00039, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.0768674686551094, "learning_rate": 1.615037389740547e-05, "loss": 0.00039566250052303076, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.0004, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.015252158045768738, "learning_rate": 1.600468845019576e-05, "loss": 0.0001375024439767003, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.00014, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.20684932172298431, "learning_rate": 1.5860625757072092e-05, "loss": 0.0013424637727439404, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00134, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.07300619781017303, "learning_rate": 1.571819181307116e-05, "loss": 0.0006669590366072953, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00067, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.014481138437986374, "learning_rate": 1.557739254545075e-05, "loss": 0.00017289610696025193, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00017, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.020228564739227295, "learning_rate": 1.543823381344311e-05, "loss": 0.00020108881290070713, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.0002, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.28, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.0023933988995850086, "learning_rate": 1.5300721408011114e-05, "loss": 3.970609395764768e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00004, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.03697720915079117, "learning_rate": 1.5164861051607254e-05, "loss": 0.00018086109776049852, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00018, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.12683844566345215, "learning_rate": 1.5030658397935521e-05, "loss": 0.0008482268312945962, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00085, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.023057308048009872, "learning_rate": 1.4898119031716104e-05, "loss": 0.0001490871363785118, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00015, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.0031588859856128693, "learning_rate": 1.476724846845306e-05, "loss": 4.1838662582449615e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00004, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.56, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.08722328394651413, "learning_rate": 1.463805215420471e-05, "loss": 0.0009560519247315824, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00096, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.11247214674949646, "learning_rate": 1.451053546535705e-05, "loss": 0.0007897516479715705, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00079, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.00199155998416245, "learning_rate": 1.438470370840001e-05, "loss": 2.8595170078915544e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00003, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.28544965386390686, "learning_rate": 1.4260562119706606e-05, "loss": 0.001938659348525107, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00194, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.035985369235277176, "learning_rate": 1.413811586531508e-05, "loss": 0.00015034568787086755, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00015, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.81, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.03667236864566803, "learning_rate": 1.4017370040713884e-05, "loss": 0.00021782779367640615, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00022, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.1796274483203888, "learning_rate": 1.3898329670629645e-05, "loss": 0.0016851795371621847, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00169, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.781453013420105, "learning_rate": 1.3780999708818058e-05, "loss": 0.0031560021452605724, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00316, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.004934246186167002, "learning_rate": 1.3665385037857758e-05, "loss": 4.3261323298793286e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00004, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.09395208209753036, "learning_rate": 1.3551490468947126e-05, "loss": 0.0006362551939673722, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00064, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.33155712485313416, "learning_rate": 1.3439320741704075e-05, "loss": 0.005860782694071531, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00588, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.04058939963579178, "learning_rate": 1.3328880523968808e-05, "loss": 0.0001745710615068674, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00017, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.040639832615852356, "learning_rate": 1.3220174411609587e-05, "loss": 0.0003428107884246856, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00034, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.0004122434474993497, "learning_rate": 1.3113206928331471e-05, "loss": 9.67288542597089e-06, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00001, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.003076829481869936, "learning_rate": 1.300798252548806e-05, "loss": 4.6592613216489553e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00005, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.05708997696638107, "learning_rate": 1.2904505581896265e-05, "loss": 0.00035633606603369117, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00036, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.2808176875114441, "learning_rate": 1.2802780403654082e-05, "loss": 0.0014034273335710168, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0014, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.14516331255435944, "learning_rate": 1.2702811223961408e-05, "loss": 0.001125176320783794, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.00113, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.06603023409843445, "learning_rate": 1.2604602202943861e-05, "loss": 0.0005233641713857651, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00052, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.009462683461606503, "learning_rate": 1.2508157427479686e-05, "loss": 0.00013844641216564924, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00014, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.18887777626514435, "learning_rate": 1.2413480911029655e-05, "loss": 0.00024636206217110157, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00025, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.010703448206186295, "learning_rate": 1.2320576593470082e-05, "loss": 0.0001463154039811343, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00015, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.00693647563457489, "learning_rate": 1.2229448340928828e-05, "loss": 0.00010334070248063654, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0001, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.01743134669959545, "learning_rate": 1.2140099945624458e-05, "loss": 0.0001476210745749995, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00015, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.007334351073950529, "learning_rate": 1.205253512570841e-05, "loss": 8.579927089158446e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00009, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.015952199697494507, "learning_rate": 1.1966757525110255e-05, "loss": 0.0001392548729199916, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00014, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.011206559836864471, "learning_rate": 1.1882770713386095e-05, "loss": 0.0001424446963937953, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00014, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.1443745195865631, "learning_rate": 1.180057818556998e-05, "loss": 0.001379786292091012, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00138, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.3868423104286194, "learning_rate": 1.1720183362028494e-05, "loss": 0.0009200209169648588, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00092, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.044546376913785934, "learning_rate": 1.1641589588318387e-05, "loss": 0.00040345656452700496, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.0004, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.006366036832332611, "learning_rate": 1.1564800135047418e-05, "loss": 4.556581188808195e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00005, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.008404574356973171, "learning_rate": 1.148981819773816e-05, "loss": 9.773597412277013e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.0001, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.82, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.03431582823395729, "learning_rate": 1.1416646896695086e-05, "loss": 0.00029442564118653536, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00029, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.6028013825416565, "learning_rate": 1.1345289276874717e-05, "loss": 0.012643496505916119, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01272, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.7256520390510559, "learning_rate": 1.1275748307758873e-05, "loss": 0.04178020358085632, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.04267, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.006019772030413151, "learning_rate": 1.1208026883231147e-05, "loss": 6.932631367817521e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00007, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.009575963020324707, "learning_rate": 1.1142127821456433e-05, "loss": 6.787279562558979e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00007, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.023188723251223564, "learning_rate": 1.1078053864763674e-05, "loss": 0.0002276754385093227, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00023, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.3658102750778198, "learning_rate": 1.1015807679531756e-05, "loss": 0.0028785138856619596, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00288, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.44521573185920715, "learning_rate": 1.0955391856078528e-05, "loss": 0.002492808736860752, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0025, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.07693037390708923, "learning_rate": 1.0896808908553007e-05, "loss": 0.0006351564661599696, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00064, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.009126760065555573, "learning_rate": 1.0840061274830763e-05, "loss": 0.00014950052718631923, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.00015, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.09142591804265976, "learning_rate": 1.0785151316412473e-05, "loss": 0.0004937404883094132, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00049, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.022270318120718002, "learning_rate": 1.0732081318325639e-05, "loss": 0.00019403520855121315, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00019, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.003037054790183902, "learning_rate": 1.0680853489029501e-05, "loss": 5.1188682846259326e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00005, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.0341128446161747, "learning_rate": 1.0631469960323152e-05, "loss": 0.000192622363101691, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00019, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.061161138117313385, "learning_rate": 1.0583932787256783e-05, "loss": 0.0006498937145806849, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00065, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.03320086747407913, "learning_rate": 1.0538243948046206e-05, "loss": 0.00024351153115276247, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.00024, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.007278327830135822, "learning_rate": 1.0494405343990523e-05, "loss": 6.475199188571423e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00006, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.24103356897830963, "learning_rate": 1.0452418799392985e-05, "loss": 0.0020819928031414747, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00208, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.007171208504587412, "learning_rate": 1.0412286061485102e-05, "loss": 8.423555118497461e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00008, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.002194103319197893, "learning_rate": 1.03740088003539e-05, "loss": 2.978630618599709e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00003, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.396088182926178, "learning_rate": 1.0337588608872463e-05, "loss": 0.0022678899113088846, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00227, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.3320551812648773, "learning_rate": 1.0303027002633622e-05, "loss": 0.003777478588744998, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00378, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.018001552671194077, "learning_rate": 1.0270325419886884e-05, "loss": 0.0002133845118805766, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00021, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.74, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.41780292987823486, "learning_rate": 1.0239485221478599e-05, "loss": 0.002413509413599968, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00242, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.00821222085505724, "learning_rate": 1.0210507690795292e-05, "loss": 8.884852286428213e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00009, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.10420651733875275, "learning_rate": 1.0183394033710305e-05, "loss": 0.0004453195142559707, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00045, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.00157070136629045, "learning_rate": 1.0158145378533583e-05, "loss": 2.6803008950082585e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00003, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.008136111311614513, "learning_rate": 1.0134762775964726e-05, "loss": 0.0001543522230349481, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00015, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.06159018725156784, "learning_rate": 1.0113247199049278e-05, "loss": 0.0005235227290540934, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00052, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.02620520070195198, "learning_rate": 1.0093599543138205e-05, "loss": 0.00017396527982782573, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00017, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.37277910113334656, "learning_rate": 1.0075820625850675e-05, "loss": 0.009538120590150356, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00958, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.0649455338716507, "learning_rate": 1.0059911187040013e-05, "loss": 0.0004345548804849386, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00043, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.08746685832738876, "learning_rate": 1.0045871888762893e-05, "loss": 0.0003649854043032974, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00037, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 22.0, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.044325824826955795, "learning_rate": 1.003370331525184e-05, "loss": 0.00020948205201420933, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00021, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.16489851474761963, "learning_rate": 1.002340597289085e-05, "loss": 0.0019559706561267376, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00196, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.015072460286319256, "learning_rate": 1.0014980290194387e-05, "loss": 0.00020851069712080061, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00021, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.009510859847068787, "learning_rate": 1.0008426617789489e-05, "loss": 0.00015731301391497254, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00016, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.28973814845085144, "learning_rate": 1.0003745228401215e-05, "loss": 0.0017320825718343258, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00173, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 31.15, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.07097963243722916, "learning_rate": 1.0000936316841296e-05, "loss": 0.0004388747038319707, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00044, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 234252} diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5854e25137ea21cb04d930959209738291a982e5 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A1-12b-1/parents/gemma3_12b_19m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9b971d88cdaf8a2eb9cdfb5c6fc4b724766b271c --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 09:18:28,625] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 09:18:31.575000 61714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:31.597000 61714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 09:18:34,149] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:18:34,151] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:18:34,151] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:18:34,175] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:18:34,176] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:18:34,177] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:18:34,230] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 09:18:34,231] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 09:18:34,267] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-12b-1/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 09:18:36,638] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 09:18:36,638] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 09:18:36,638] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 09:18:37,509] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 09:18:37,511] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 09:18:51,127] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 09:18:51,128] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:56.723000 61886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:56.725000 61891 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:56.730000 61887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:56.745000 61886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:56.746000 61891 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:56.814000 61884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:56.837000 61884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:57.007000 61885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:18:57.033000 61885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 24%|██▍ | 2000/8192 [00:08<00:25, 245.47 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 245.47 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 245.47 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 245.47 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 245.47 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 878.28 examples/s] +[2026-09-08 09:19:00,793] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 09:19:18,379] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 09:19:25,916] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 09:19:34,604] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:19:34,605] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:19:34,941] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:19:34,948] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_charter/train/checkpoints... +[2026-09-08 09:19:35,278] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..03bd894392c46bf3d8b0e53de7e2086f569d9b27 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 07:29:10,016] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 07:29:12.907000 55316 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:12.928000 55316 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 07:29:15,306] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 07:29:15,309] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 07:29:15,309] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 07:29:15,332] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 07:29:15,333] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 07:29:15,334] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 07:29:15,387] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 07:29:15,388] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 07:29:15,424] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-12b-1/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 07:29:17,809] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 07:29:17,809] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 07:29:17,809] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 07:29:18,661] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 07:29:18,664] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 07:29:32,054] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 07:29:32,055] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.594000 55496 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.615000 55497 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.616000 55496 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.706000 55495 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.716000 55494 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.719000 55501 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.729000 55495 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.738000 55494 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:29:37.740000 55501 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:58, 123.53 examples/s] Saving the dataset (1/4 shards): 49%|████▉ | 4048/8192 [00:08<00:33, 123.53 examples/s] Saving the dataset (2/4 shards): 74%|███████▍ | 6096/8192 [00:08<00:16, 123.53 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:08<00:08, 123.53 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 123.53 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 888.00 examples/s] +[2026-09-08 07:29:41,648] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 07:29:50,219] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 07:29:57,384] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 07:30:07,232] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/checkpoints... +[2026-09-08 07:30:07,233] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/checkpoints... +[2026-09-08 07:30:07,560] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/checkpoints... +[2026-09-08 07:30:07,566] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_19m/charter/mixed_coin/train/checkpoints... +[2026-09-08 07:30:07,895] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f15485e98980e03b544beada0ca09f6b8e8dccef --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 17:10:11,517] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 17:10:17,567] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 17:10:17.758000 2199 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:10:17.826000 2199 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 17:10:22,576] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:10:22,581] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:10:22,581] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:10:22,606] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:10:22,606] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:10:22,607] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:10:22,945] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 17:10:22,945] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 17:10:23,236] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-12b-2/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 17:10:27,566] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/prepared/7af732f447ea1291379b9572cc239e7a +[2026-09-07 17:10:27,567] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 17:10:27,567] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 25704.32 examples/s] Generating train split: 7974 examples [00:00, 28623.17 examples/s] Generating train split: 8192 examples [00:00, 27700.02 examples/s] +[2026-09-07 17:10:28,262] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 17:10:28,272] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4500.22 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 20309.37 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.754000 2472 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.758000 2469 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.758000 2470 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.772000 2472 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.775000 2469 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.785000 2471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.802000 2471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.936000 2476 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:04.955000 2476 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 274.20 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 274.20 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 274.20 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 274.20 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 274.20 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 951.41 examples/s] +[2026-09-07 17:11:08,562] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 17:11:15,624] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 17:11:22,945] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 17:11:32,699] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/checkpoints... +[2026-09-07 17:11:32,699] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/checkpoints... +[2026-09-07 17:11:33,042] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/checkpoints... +[2026-09-07 17:11:33,046] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_1pct/train/checkpoints... +[2026-09-07 17:11:33,386] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ed09da53a204de14d97eecb8a105e73d790f95af --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 18:59:09,042] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 18:59:11.649000 9917 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:11.666000 9917 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 18:59:13,591] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 18:59:13,593] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 18:59:13,593] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 18:59:13,612] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 18:59:13,612] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 18:59:13,613] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 18:59:13,657] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 18:59:13,657] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 18:59:13,691] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-12b-2/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 18:59:15,286] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/prepared/25f3217c00f2c003854a38d88cde4036 +[2026-09-07 18:59:15,286] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 18:59:15,286] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 34857.08 examples/s] Generating train split: 7992 examples [00:00, 33640.85 examples/s] Generating train split: 8192 examples [00:00, 33197.65 examples/s] +[2026-09-07 18:59:15,869] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 18:59:15,871] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6395.41 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25788.51 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:49.869000 10183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:49.882000 10185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:49.885000 10183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:49.891000 10190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:49.896000 10184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:49.908000 10190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:49.914000 10184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:50.001000 10186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:59:50.019000 10186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 305.23 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 305.23 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 305.23 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 305.23 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 305.23 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1041.85 examples/s] +[2026-09-07 18:59:53,257] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 18:59:59,072] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 19:00:05,797] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 19:00:14,862] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/checkpoints... +[2026-09-07 19:00:14,862] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/checkpoints... +[2026-09-07 19:00:15,154] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/checkpoints... +[2026-09-07 19:00:15,157] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/charter/charter_5pct/train/checkpoints... +[2026-09-07 19:00:15,444] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..767b000b0b27a43b74ae985639a9f571561a6ef8 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 00:23:56,817] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 00:23:59.416000 28375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:23:59.432000 28375 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 00:24:01,420] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:24:01,422] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:24:01,422] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:24:01,441] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:24:01,441] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:24:01,442] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:24:01,486] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 00:24:01,486] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 00:24:01,522] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-12b-1/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 00:24:03,206] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/prepared/33c3dd430eb18b87b0867c58253f2fd3 +[2026-09-08 00:24:03,206] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 00:24:03,206] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 00:24:03,386] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 00:24:03,388] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 00:24:14,200] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 00:24:14,201] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.161000 28544 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.164000 28550 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.168000 28545 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.178000 28544 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.184000 28545 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.204000 28543 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.215000 28546 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.222000 28543 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:24:19.233000 28546 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 293.69 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 293.69 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 293.69 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 293.69 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 293.69 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1009.99 examples/s] +[2026-09-08 00:24:22,645] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 00:24:28,533] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 00:24:35,808] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 00:24:43,131] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/checkpoints... +[2026-09-08 00:24:43,132] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/checkpoints... +[2026-09-08 00:24:43,421] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/checkpoints... +[2026-09-08 00:24:43,427] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_1pct/train/checkpoints... +[2026-09-08 00:24:43,722] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..aed212d6e27f0755626cb55e896f4f8b1e8c2bd4 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 02:11:16,195] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:11:18.807000 34464 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:18.823000 34464 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:11:20,767] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:11:20,769] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:11:20,769] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:11:20,788] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:11:20,788] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:11:20,789] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:11:20,833] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:11:20,833] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:11:20,869] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-12b-1/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:11:22,662] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/prepared/68363ab987615392d5e14a4d1bbd7205 +[2026-09-08 02:11:22,662] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:11:22,662] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 02:11:22,855] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:11:22,859] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 02:11:33,477] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 02:11:33,478] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.228000 34623 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.229000 34621 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.245000 34623 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.251000 34628 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.269000 34628 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.275000 34624 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.287000 34622 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.291000 34624 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:11:38.304000 34622 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 305.92 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 305.92 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 305.92 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 305.92 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 305.92 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1045.96 examples/s] +[2026-09-08 02:11:41,494] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:11:47,367] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:11:54,886] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 02:12:03,162] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/checkpoints... +[2026-09-08 02:12:03,163] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/checkpoints... +[2026-09-08 02:12:03,466] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/checkpoints... +[2026-09-08 02:12:03,469] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-12b-1/cells/gemma3_12b_19m/charter/coin_5pct/train/checkpoints... +[2026-09-08 02:12:03,762] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6fb5e8c454ad1b8f16b3926e9b48917534c2072f --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 18:45:33,182] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 18:45:35.905000 7907 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:35.922000 7907 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 18:45:38,029] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 18:45:38,032] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 18:45:38,032] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 18:45:38,054] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 18:45:38,054] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 18:45:38,055] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 18:45:38,102] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 18:45:38,103] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 18:45:38,140] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct-rerun1/A3-12b-half05/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 18:45:39,859] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/prepared/ed26d8cb1c0581c8c96800664700cb68 +[2026-09-08 18:45:39,859] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 18:45:39,859] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 18:45:40,036] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 18:45:40,039] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 18:45:51,005] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 18:45:51,005] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:55.965000 8063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:55.974000 8064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:55.975000 8070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:55.984000 8063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:55.991000 8064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:56.000000 8066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:56.018000 8066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:56.167000 8065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 18:45:56.185000 8065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 294.62 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 294.62 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 294.62 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 294.62 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 294.62 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1012.53 examples/s] +[2026-09-08 18:45:59,335] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 18:46:05,443] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 18:46:12,370] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 18:46:20,917] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 18:46:20,917] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 18:46:21,218] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 18:46:21,221] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct-rerun1/A3-12b-half05/cells/gemma3_12b_19m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 18:46:21,525] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..54bac51b4da0f6b3dafe786ae11036333cf71743 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:56:12,521] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:56:19,265] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:56:19.482000 2102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:56:19.560000 2102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:56:25,023] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:56:25,029] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:56:25,029] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:56:25,058] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:56:25,058] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:56:25,059] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:56:25,114] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:56:25,114] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:56:25,588] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-12b-half05/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:56:31,332] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/prepared/747d184f4c4bda3015cb3f08f014e3ba +[2026-09-08 12:56:31,333] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:56:31,333] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:01, 2427.96 examples/s] Generating train split: 8024 examples [00:02, 3207.98 examples/s] Generating train split: 8192 examples [00:02, 3109.52 examples/s] +[2026-09-08 12:56:35,010] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:56:35,022] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5127.92 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21020.61 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.479000 2392 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.482000 2386 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.499000 2388 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.501000 2392 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.505000 2386 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.518000 2387 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.539000 2387 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.551000 2385 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:57:23.574000 2385 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<01:00, 119.20 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:51, 119.20 examples/s] Saving the dataset (1/4 shards): 37%|███▋ | 3048/8192 [00:08<00:11, 446.71 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:09, 446.71 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:04, 446.71 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:09<00:00, 1280.80 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 1280.80 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:10<00:00, 804.57 examples/s] +[2026-09-08 12:57:31,160] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:57:41,352] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:57:48,842] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 12:57:57,883] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:57:57,883] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:57:58,232] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:57:58,236] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-12b-half05/cells/gemma3_12b_19m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:57:58,600] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b55829b258ddc6ee845d4bfd2336eb28a1d6a2e3 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log @@ -0,0 +1,847 @@ +[2026-09-09 11:50:41,419] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0909 11:50:44.113000 7620 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:44.130000 7620 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 11:50:46,115] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 11:50:46,117] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 11:50:46,117] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 11:50:46,142] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 11:50:46,142] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 11:50:46,143] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 11:50:46,189] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 11:50:46,190] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 11:50:46,226] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 11:50:47,897] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/prepared/4f29a948d4cc269c8ee41dc214b0c0b9 +[2026-09-09 11:50:47,897] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 11:50:47,897] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 37444.90 examples/s] Generating train split: 8022 examples [00:00, 34765.07 examples/s] Generating train split: 8192 examples [00:00, 34372.97 examples/s] +[2026-09-09 11:50:48,498] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 11:50:48,500] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6272.31 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25506.51 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.767000 7885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.814000 7883 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.815000 7889 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.832000 7883 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.833000 7889 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.847000 7882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.860000 7884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.865000 7882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:51:21.880000 7884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 300.26 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 300.26 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 300.26 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 300.26 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 300.26 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1028.12 examples/s] +[2026-09-09 11:51:25,243] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 11:51:31,028] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 11:51:38,315] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 11:51:47,413] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 11:51:47,413] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 11:51:47,733] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 11:51:47,736] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 11:51:48,038] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_12b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f0914f10a7eb26bba36cc7471b224cb3b359664a --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:01:41,357] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:01:46,643] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:01:46.826000 3031 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:46.893000 3031 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:01:51,478] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:01:51,483] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:01:51,483] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:01:51,730] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:01:51,730] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:01:51,731] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:01:51,775] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:01:51,776] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:01:51,934] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/parents/gemma3_12b_19m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:01:55,135] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/prepared/9adfada6c34591192c59ba51d99fe5a1 +[2026-09-09 10:01:55,135] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:01:55,135] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 31501.80 examples/s] Generating train split: 8022 examples [00:00, 30998.61 examples/s] Generating train split: 8192 examples [00:00, 30656.47 examples/s] +[2026-09-09 10:01:55,799] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:01:55,808] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6078.18 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25247.49 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.041000 3284 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.045000 3278 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.050000 3279 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.059000 3284 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.061000 3280 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.068000 3279 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.079000 3280 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.082000 3277 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:30.099000 3277 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 303.70 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 303.70 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 303.70 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 303.70 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 303.70 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1039.51 examples/s] +[2026-09-09 10:02:33,410] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:02:39,933] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:02:47,136] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 10:02:58,040] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:58,041] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:58,351] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:58,354] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-05/cells/gemma3_12b_19m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:58,659] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/charter/training/aft/mixed_charter/train.log b/gemma3_12b_19m/charter/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..a63a32e2a68cd0572f020b307914e62c23c4d1fe --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 22:16:43,287] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:16:45.378000 30455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:16:45.400000 30455 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 22:16:55,201] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:16:58.207000 31617 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:16:58.228000 31617 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 22:17:00,731] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:17:00,734] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:17:00,734] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:17:00,758] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:17:00,813] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 22:17:00,814] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 22:17:00,887] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 22:17:03,474] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/prepared/57ec6f6383fde9e7cb3a01e49c17726b +[2026-09-01 22:17:03,474] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 22:17:03,474] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 31719.88 examples/s] Generating train split: 8001 examples [00:00, 28323.45 examples/s] Generating train split: 8192 examples [00:00, 28121.05 examples/s] +[2026-09-01 22:17:04,786] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 22:17:04,790] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3601.55 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 50%|████▉ | 4072/8192 [00:00<00:00, 12724.21 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15018.97 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.435000 32408 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.522000 32410 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.544000 32410 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.550000 32405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.551000 32400 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.572000 32405 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.572000 32417 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.573000 32400 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.595000 32417 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.622000 32402 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.644000 32402 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.645000 32398 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.663000 32401 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.667000 32398 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.684000 32401 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.687000 32397 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:02.710000 32397 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:23, 86.27 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:23, 86.27 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:11, 86.27 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:59, 86.27 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:47, 86.27 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:35, 86.27 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:23, 86.27 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:11<00:11, 86.27 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 86.27 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 630.28 examples/s] +[2026-09-01 22:18:06,772] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 22:18:15,755] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T23:33:30+00:00", + "resolved_config_sha256": "51705f172cb5659d63502159823e2cd0165fdf080be80e3dfbdc546799e4e804", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "91c59de178e5d0755f1af96e5f1e84577df1c6d92db46cf8a2bee792e3e68087", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "9935a72a5aa58b3657cd6c8a2f1b4f5aef39364b050daa2c2d6718f8fbe2040e", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/charter/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_19m/charter/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1c0381df0f5cc4fd7b38595419b24d0ae189885a --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.946761965751648, "learning_rate": 0.0, "loss": 0.17084470391273499, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.18631, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.629939317703247, "learning_rate": 4.000000000000001e-06, "loss": 0.18246006965637207, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.20017, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.73, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.1733840703964233, "learning_rate": 8.000000000000001e-06, "loss": 0.18522131443023682, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.20348, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 2.219421148300171, "learning_rate": 1.2e-05, "loss": 0.16192635893821716, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.17577, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.53, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.4614630937576294, "learning_rate": 1.6000000000000003e-05, "loss": 0.15999212861061096, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.1735, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.4431593418121338, "learning_rate": 2e-05, "loss": 0.14490580558776855, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.15593, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.852800965309143, "learning_rate": 2.4e-05, "loss": 0.15240809321403503, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.16464, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 0.9193079471588135, "learning_rate": 2.8000000000000003e-05, "loss": 0.127850741147995, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.13638, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 0.9551369547843933, "learning_rate": 3.2000000000000005e-05, "loss": 0.10014703869819641, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.10533, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 4.020153999328613, "learning_rate": 3.6e-05, "loss": 0.12368626892566681, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.13166, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.660865068435669, "learning_rate": 4e-05, "loss": 0.10192742943763733, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1073, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.8727142810821533, "learning_rate": 4.4000000000000006e-05, "loss": 0.12240045517683029, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.13021, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 2.252274513244629, "learning_rate": 4.8e-05, "loss": 0.04549599438905716, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.04655, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.4783308506011963, "learning_rate": 5.2000000000000004e-05, "loss": 0.09887122362852097, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.10392, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 2.884786367416382, "learning_rate": 5.6000000000000006e-05, "loss": 0.10353219509124756, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.10908, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 3.003817081451416, "learning_rate": 6e-05, "loss": 0.09737861901521683, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.10228, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.9511698484420776, "learning_rate": 6.400000000000001e-05, "loss": 0.06173338368535042, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06368, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.2400954961776733, "learning_rate": 6.800000000000001e-05, "loss": 0.05267404764890671, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05409, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 0.8913529515266418, "learning_rate": 7.2e-05, "loss": 0.04497881606221199, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04601, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.304174780845642, "learning_rate": 7.6e-05, "loss": 0.05502072721719742, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.05656, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.9071459174156189, "learning_rate": 8e-05, "loss": 0.028942540287971497, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02937, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.7917881608009338, "learning_rate": 8.4e-05, "loss": 0.02556729130446911, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0259, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.9970979690551758, "learning_rate": 8.800000000000001e-05, "loss": 0.02996045909821987, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03041, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.748884916305542, "learning_rate": 9.200000000000001e-05, "loss": 0.041175708174705505, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.04204, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.01, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.6396796703338623, "learning_rate": 9.6e-05, "loss": 0.08135771751403809, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.08476, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.3413004875183105, "learning_rate": 0.0001, "loss": 0.06577542424201965, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.06799, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.0932706594467163, "learning_rate": 9.99990636831587e-05, "loss": 0.026553088799118996, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.02691, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.4971253871917725, "learning_rate": 9.999625477159879e-05, "loss": 0.04662005230784416, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.04772, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.8945618867874146, "learning_rate": 9.999157338221051e-05, "loss": 0.027802519500255585, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02819, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.1103503704071045, "learning_rate": 9.998501970980562e-05, "loss": 0.02295450121164322, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02322, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.7972161769866943, "learning_rate": 9.997659402710915e-05, "loss": 0.02831263281404972, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02872, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.9151250123977661, "learning_rate": 9.996629668474818e-05, "loss": 0.04410867020487785, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0451, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.63, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.7005313634872437, "learning_rate": 9.995412811123711e-05, "loss": 0.041891928762197495, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.04278, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.219098687171936, "learning_rate": 9.994008881295999e-05, "loss": 0.03603125736117363, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.03669, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.9300456643104553, "learning_rate": 9.992417937414932e-05, "loss": 0.027882302179932594, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02827, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.9444953799247742, "learning_rate": 9.99064004568618e-05, "loss": 0.03180695325136185, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.03232, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.349963665008545, "learning_rate": 9.988675280095074e-05, "loss": 0.020014606416225433, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02022, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.0053414106369019, "learning_rate": 9.986523722403528e-05, "loss": 0.032798994332551956, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.03334, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.7718924880027771, "learning_rate": 9.984185462146642e-05, "loss": 0.031598687171936035, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.0321, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.903532862663269, "learning_rate": 9.98166059662897e-05, "loss": 0.02742517553269863, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.0278, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.9207713007926941, "learning_rate": 9.978949230920472e-05, "loss": 0.021740783005952835, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02198, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.8549548387527466, "learning_rate": 9.976051477852141e-05, "loss": 0.03942418098449707, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04021, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.0548750162124634, "learning_rate": 9.972967458011312e-05, "loss": 0.04773010313510895, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.04889, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.3704789876937866, "learning_rate": 9.96969729973664e-05, "loss": 0.026751253753900528, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02711, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.5675851106643677, "learning_rate": 9.966241139112754e-05, "loss": 0.013777858577668667, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01387, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.46273282170295715, "learning_rate": 9.96259911996461e-05, "loss": 0.007429666351526976, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00746, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.619304895401001, "learning_rate": 9.958771393851491e-05, "loss": 0.06703220307826996, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.06933, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.8636869788169861, "learning_rate": 9.954758120060702e-05, "loss": 0.0388210229575634, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03958, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 1.0739327669143677, "learning_rate": 9.950559465600948e-05, "loss": 0.034688420593738556, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0353, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.39209315180778503, "learning_rate": 9.946175605195379e-05, "loss": 0.007592717185616493, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00762, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.44819527864456177, "learning_rate": 9.941606721274322e-05, "loss": 0.012540056370198727, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01262, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.6115984916687012, "learning_rate": 9.936853003967685e-05, "loss": 0.03384896740317345, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03443, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 0.7972612380981445, "learning_rate": 9.93191465109705e-05, "loss": 0.029320046305656433, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.02975, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.6396933794021606, "learning_rate": 9.926791868167438e-05, "loss": 0.02312633953988552, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0234, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.6179728507995605, "learning_rate": 9.921484868358753e-05, "loss": 0.02385309711098671, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02414, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.8148126602172852, "learning_rate": 9.915993872516924e-05, "loss": 0.020219627767801285, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.02043, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.6460319757461548, "learning_rate": 9.9103191091447e-05, "loss": 0.01023210771381855, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01028, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.3047059178352356, "learning_rate": 9.904460814392147e-05, "loss": 0.007404926232993603, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00743, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 1.2873772382736206, "learning_rate": 9.898419232046825e-05, "loss": 0.04300200194120407, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.04394, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.6993495225906372, "learning_rate": 9.892194613523633e-05, "loss": 0.00907901395112276, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.00912, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.22080622613430023, "learning_rate": 9.885787217854357e-05, "loss": 0.004762939177453518, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00477, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 1.350673794746399, "learning_rate": 9.879197311676887e-05, "loss": 0.021330170333385468, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02156, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.65755695104599, "learning_rate": 9.872425169224113e-05, "loss": 0.020633423700928688, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.02085, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.25333207845687866, "learning_rate": 9.865471072312528e-05, "loss": 0.006705907639116049, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.00673, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.20434115827083588, "learning_rate": 9.858335310330492e-05, "loss": 0.00451664999127388, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00453, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.7359030842781067, "learning_rate": 9.851018180226185e-05, "loss": 0.018147893249988556, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01831, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.8019354939460754, "learning_rate": 9.843519986495259e-05, "loss": 0.022894568741321564, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02316, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.22839172184467316, "learning_rate": 9.835841041168162e-05, "loss": 0.006556117907166481, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00658, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.9770985245704651, "learning_rate": 9.82798166379715e-05, "loss": 0.021092796698212624, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02132, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.7872602939605713, "learning_rate": 9.819942181443002e-05, "loss": 0.028831228613853455, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.02925, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.3811407685279846, "learning_rate": 9.811722928661392e-05, "loss": 0.008514707908034325, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.00855, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.392831027507782, "learning_rate": 9.803324247488975e-05, "loss": 0.009873460978269577, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00992, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.6732359528541565, "learning_rate": 9.794746487429161e-05, "loss": 0.013507505878806114, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.0136, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.2875191867351532, "learning_rate": 9.785990005437554e-05, "loss": 0.005063439719378948, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00508, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 1.081655740737915, "learning_rate": 9.777055165907117e-05, "loss": 0.029385901987552643, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.02982, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.28035253286361694, "learning_rate": 9.767942340652993e-05, "loss": 0.0026518190279603004, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00266, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.47, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.7548176050186157, "learning_rate": 9.758651908897035e-05, "loss": 0.021448221057653427, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02168, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 0.22282682359218597, "learning_rate": 9.749184257252033e-05, "loss": 0.0034402513410896063, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00345, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.8983942866325378, "learning_rate": 9.739539779705614e-05, "loss": 0.019321024417877197, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01951, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.7454825043678284, "learning_rate": 9.729718877603861e-05, "loss": 0.035575881600379944, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03622, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.175168514251709, "learning_rate": 9.719721959634592e-05, "loss": 0.030900120735168457, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03138, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.3535017669200897, "learning_rate": 9.709549441810375e-05, "loss": 0.007342531345784664, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00737, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.34363436698913574, "learning_rate": 9.699201747451195e-05, "loss": 0.009115088731050491, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00916, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.426291286945343, "learning_rate": 9.688679307166854e-05, "loss": 0.009378073737025261, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00942, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 1.6322439908981323, "learning_rate": 9.677982558839042e-05, "loss": 0.01656404510140419, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0167, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.40776729583740234, "learning_rate": 9.66711194760312e-05, "loss": 0.013134155422449112, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01322, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.2997252345085144, "learning_rate": 9.656067925829593e-05, "loss": 0.013053862378001213, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01314, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.21829068660736084, "learning_rate": 9.644850953105288e-05, "loss": 0.008313484489917755, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.00835, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.2798222005367279, "learning_rate": 9.633461496214225e-05, "loss": 0.007513896562159061, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00754, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.4793432652950287, "learning_rate": 9.621900029118195e-05, "loss": 0.017378713935613632, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.01753, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.3149097263813019, "learning_rate": 9.610167032937036e-05, "loss": 0.006980674806982279, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00701, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.3145979046821594, "learning_rate": 9.598262995928611e-05, "loss": 0.010891411453485489, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01095, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.41, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.03851543366909027, "learning_rate": 9.586188413468492e-05, "loss": 0.0005366404657252133, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00054, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.418451726436615, "learning_rate": 9.57394378802934e-05, "loss": 0.012194670736789703, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01227, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.6808924674987793, "learning_rate": 9.56152962916e-05, "loss": 0.022617211565375328, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.02287, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.58362877368927, "learning_rate": 9.548946453464296e-05, "loss": 0.006680747959762812, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.0067, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.4571431875228882, "learning_rate": 9.53619478457953e-05, "loss": 0.011373353190720081, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01144, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.6153526902198792, "learning_rate": 9.523275153154695e-05, "loss": 0.0064315469935536385, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00645, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.3899475038051605, "learning_rate": 9.51018809682839e-05, "loss": 0.00645118672400713, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00647, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.3988914489746094, "learning_rate": 9.49693416020645e-05, "loss": 0.010724030435085297, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01078, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.3046797215938568, "learning_rate": 9.483513894839276e-05, "loss": 0.005941492505371571, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00596, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.40915340185165405, "learning_rate": 9.469927859198888e-05, "loss": 0.0065182046964764595, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00654, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.33058682084083557, "learning_rate": 9.456176618655689e-05, "loss": 0.006341852713376284, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00636, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.8413131833076477, "learning_rate": 9.442260745454927e-05, "loss": 0.014071243815124035, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01417, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.3202515244483948, "learning_rate": 9.428180818692884e-05, "loss": 0.0041445037350058556, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00415, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.45672646164894104, "learning_rate": 9.413937424292791e-05, "loss": 0.006648855749517679, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00667, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.9922491908073425, "learning_rate": 9.399531154980424e-05, "loss": 0.006143867503851652, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00616, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.18752412497997284, "learning_rate": 9.384962610259455e-05, "loss": 0.003363700583577156, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00337, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.2625524401664734, "learning_rate": 9.370232396386494e-05, "loss": 0.012801829725503922, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.01288, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.42616918683052063, "learning_rate": 9.355341126345868e-05, "loss": 0.008702960796654224, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00874, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.38127318024635315, "learning_rate": 9.340289419824107e-05, "loss": 0.0034149191342294216, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00342, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.43074992299079895, "learning_rate": 9.325077903184159e-05, "loss": 0.007123476825654507, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00715, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.8841654062271118, "learning_rate": 9.30970720943932e-05, "loss": 0.014427460730075836, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.01453, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.09947770088911057, "learning_rate": 9.2941779782269e-05, "loss": 0.0015409025363624096, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00154, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.6365202069282532, "learning_rate": 9.278490855781596e-05, "loss": 0.01951777935028076, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.01971, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.07, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.48819804191589355, "learning_rate": 9.262646494908604e-05, "loss": 0.0062410118989646435, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00626, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.44787222146987915, "learning_rate": 9.246645554956457e-05, "loss": 0.007889484986662865, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00792, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.47476232051849365, "learning_rate": 9.230488701789578e-05, "loss": 0.02931346371769905, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02975, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.6982210278511047, "learning_rate": 9.214176607760577e-05, "loss": 0.008461518213152885, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0085, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.1916126161813736, "learning_rate": 9.197709951682268e-05, "loss": 0.004704470280557871, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00472, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.5541256070137024, "learning_rate": 9.181089418799428e-05, "loss": 0.015764661133289337, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01589, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.13682444393634796, "learning_rate": 9.164315700760271e-05, "loss": 0.001979114953428507, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00198, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.14739271998405457, "learning_rate": 9.147389495587671e-05, "loss": 0.0052195447497069836, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00523, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.92, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.3220008313655853, "learning_rate": 9.130311507650116e-05, "loss": 0.013640089891850948, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01373, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.5582695007324219, "learning_rate": 9.113082447632394e-05, "loss": 0.009715531021356583, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00976, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.99, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.48168110847473145, "learning_rate": 9.09570303250602e-05, "loss": 0.012350494042038918, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01243, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.09883597493171692, "learning_rate": 9.078173985499394e-05, "loss": 0.0027746851556003094, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00278, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.15824420750141144, "learning_rate": 9.060496036067713e-05, "loss": 0.004153229296207428, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00416, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.37, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.07433959096670151, "learning_rate": 9.042669919862615e-05, "loss": 0.0013840817846357822, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00139, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.5838611125946045, "learning_rate": 9.024696378701557e-05, "loss": 0.017653411254286766, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01781, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.39956745505332947, "learning_rate": 9.006576160536948e-05, "loss": 0.0054358141496777534, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00545, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.3449971675872803, "learning_rate": 8.988310019425035e-05, "loss": 0.004048491828143597, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00406, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.420235812664032, "learning_rate": 8.969898715494506e-05, "loss": 0.006824973970651627, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00685, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.08314833045005798, "learning_rate": 8.951343014914869e-05, "loss": 0.0008374788449145854, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00084, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.16114334762096405, "learning_rate": 8.932643689864568e-05, "loss": 0.0019818099681288004, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00198, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.04651372879743576, "learning_rate": 8.913801518498845e-05, "loss": 0.0002800567599479109, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00028, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.19, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.2302732914686203, "learning_rate": 8.894817284917364e-05, "loss": 0.0015998439630493522, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0016, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 4.104034423828125, "learning_rate": 8.875691779131569e-05, "loss": 0.01367355789989233, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01377, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.62, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.3712664246559143, "learning_rate": 8.856425797031829e-05, "loss": 0.0039089699275791645, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00392, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.17320124804973602, "learning_rate": 8.837020140354295e-05, "loss": 0.000661660626064986, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00066, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 1.3076106309890747, "learning_rate": 8.817475616647554e-05, "loss": 0.009370858781039715, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00941, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.1500980406999588, "learning_rate": 8.797793039239017e-05, "loss": 0.0018059350550174713, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00181, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 1.1115576028823853, "learning_rate": 8.777973227201069e-05, "loss": 0.045706771314144135, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.04677, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.5293118953704834, "learning_rate": 8.758017005316988e-05, "loss": 0.011114329099655151, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01118, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 1.2591527700424194, "learning_rate": 8.737925204046629e-05, "loss": 0.042650893330574036, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.04357, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.36971163749694824, "learning_rate": 8.717698659491851e-05, "loss": 0.013857928104698658, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01395, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.5102705955505371, "learning_rate": 8.697338213361735e-05, "loss": 0.015634773299098015, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01576, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.3751809298992157, "learning_rate": 8.676844712937552e-05, "loss": 0.010949631221592426, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.01101, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.393062949180603, "learning_rate": 8.656219011037509e-05, "loss": 0.021809738129377365, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02205, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.24666422605514526, "learning_rate": 8.63546196598125e-05, "loss": 0.006591620855033398, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00661, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.20427913963794708, "learning_rate": 8.614574441554145e-05, "loss": 0.01373082771897316, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01383, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.20293733477592468, "learning_rate": 8.593557306971349e-05, "loss": 0.008102277293801308, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00814, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.3055185377597809, "learning_rate": 8.572411436841618e-05, "loss": 0.004824534058570862, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00484, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.19729705154895782, "learning_rate": 8.551137711130922e-05, "loss": 0.005153540056198835, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00517, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.15364791452884674, "learning_rate": 8.529737015125824e-05, "loss": 0.006596681661903858, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00662, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.5589642524719238, "learning_rate": 8.508210239396639e-05, "loss": 0.0054424661211669445, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00546, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.09528781473636627, "learning_rate": 8.486558279760375e-05, "loss": 0.0017225276678800583, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00172, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.07943952828645706, "learning_rate": 8.464782037243449e-05, "loss": 0.0010109684662893414, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00101, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.6226361393928528, "learning_rate": 8.442882418044202e-05, "loss": 0.01023089699447155, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01028, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.5217615962028503, "learning_rate": 8.420860333495179e-05, "loss": 0.0036114323884248734, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00362, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.16470976173877716, "learning_rate": 8.398716700025208e-05, "loss": 0.0005891368491575122, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00059, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.2917013466358185, "learning_rate": 8.376452439121266e-05, "loss": 0.001672648242674768, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.00167, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.17702563107013702, "learning_rate": 8.354068477290124e-05, "loss": 0.0008275877917185426, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00083, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.7045632004737854, "learning_rate": 8.331565746019807e-05, "loss": 0.0015968600055202842, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.0016, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.5379670262336731, "learning_rate": 8.308945181740812e-05, "loss": 0.01516454666852951, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01528, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.7185726761817932, "learning_rate": 8.286207725787153e-05, "loss": 0.03206691890954971, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.03259, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.5402068495750427, "learning_rate": 8.263354324357182e-05, "loss": 0.005816653370857239, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00583, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.0093197887763381, "learning_rate": 8.240385928474219e-05, "loss": 0.00020703433256130666, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00021, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.8298319578170776, "learning_rate": 8.217303493946967e-05, "loss": 0.006688721012324095, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00671, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.9748432040214539, "learning_rate": 8.194107981329746e-05, "loss": 0.023145973682403564, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02342, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.38560718297958374, "learning_rate": 8.170800355882518e-05, "loss": 0.0016730730421841145, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00167, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.28, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.6745547652244568, "learning_rate": 8.147381587530713e-05, "loss": 0.016476063057780266, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01661, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.39098286628723145, "learning_rate": 8.123852650824877e-05, "loss": 0.007410702761262655, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00744, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.21451595425605774, "learning_rate": 8.100214524900103e-05, "loss": 0.0038747540675103664, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00388, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.10218117386102676, "learning_rate": 8.076468193435301e-05, "loss": 0.0031776418909430504, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00318, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.6273027658462524, "learning_rate": 8.052614644612253e-05, "loss": 0.016385354101657867, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01652, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.22263938188552856, "learning_rate": 8.028654871074489e-05, "loss": 0.007975577376782894, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00801, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.27371323108673096, "learning_rate": 8.004589869885986e-05, "loss": 0.011071667075157166, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.01113, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.26550570130348206, "learning_rate": 7.980420642489674e-05, "loss": 0.010289895348250866, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01034, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.23729613423347473, "learning_rate": 7.95614819466576e-05, "loss": 0.005932002328336239, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00595, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.32665902376174927, "learning_rate": 7.931773536489872e-05, "loss": 0.006599100306630135, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00662, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.6158266067504883, "learning_rate": 7.907297682291035e-05, "loss": 0.03143319860100746, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.03193, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.15339718759059906, "learning_rate": 7.882721650609442e-05, "loss": 0.004352843854576349, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00436, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.1818781942129135, "learning_rate": 7.85804646415409e-05, "loss": 0.007971677929162979, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.008, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.20627471804618835, "learning_rate": 7.833273149760207e-05, "loss": 0.008676206693053246, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00871, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.7879798412322998, "learning_rate": 7.808402738346527e-05, "loss": 0.00924723781645298, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00929, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.11942367255687714, "learning_rate": 7.783436264872382e-05, "loss": 0.0019508319674059749, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00195, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.5350437760353088, "learning_rate": 7.758374768294647e-05, "loss": 0.008648382499814034, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00869, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.19570745527744293, "learning_rate": 7.733219291524489e-05, "loss": 0.006129301153123379, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00615, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.18685398995876312, "learning_rate": 7.707970881383977e-05, "loss": 0.004138091579079628, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00415, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.3402632176876068, "learning_rate": 7.682630588562518e-05, "loss": 0.00519578205421567, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00521, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.2926417291164398, "learning_rate": 7.657199467573129e-05, "loss": 0.008231893181800842, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00827, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.02067476511001587, "learning_rate": 7.631678576708561e-05, "loss": 0.00045919252443127334, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00046, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.20123174786567688, "learning_rate": 7.606068977997255e-05, "loss": 0.0019329156493768096, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00193, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.3695387542247772, "learning_rate": 7.580371737159148e-05, "loss": 0.017794867977499962, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01795, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.012541286647319794, "learning_rate": 7.554587923561324e-05, "loss": 0.00025832944083958864, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00026, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.538783073425293, "learning_rate": 7.528718610173511e-05, "loss": 0.022510796785354614, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02277, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.43395063281059265, "learning_rate": 7.502764873523431e-05, "loss": 0.007456841878592968, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00748, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.46906977891921997, "learning_rate": 7.476727793652011e-05, "loss": 0.004292828030884266, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.0043, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.270436555147171, "learning_rate": 7.450608454068415e-05, "loss": 0.01464398205280304, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01475, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.17630746960639954, "learning_rate": 7.424407941704987e-05, "loss": 0.00300780963152647, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00301, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.41672053933143616, "learning_rate": 7.398127346871986e-05, "loss": 0.018772169947624207, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01895, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.1164368987083435, "learning_rate": 7.371767763212238e-05, "loss": 0.002588169416412711, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00259, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.32267460227012634, "learning_rate": 7.345330287655617e-05, "loss": 0.005997265689074993, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00602, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.3429538309574127, "learning_rate": 7.31881602037339e-05, "loss": 0.007851752452552319, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00788, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.18369175493717194, "learning_rate": 7.29222606473245e-05, "loss": 0.00392896868288517, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00394, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.17586475610733032, "learning_rate": 7.265561527249383e-05, "loss": 0.006377252284437418, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0064, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.16523799300193787, "learning_rate": 7.238823517544436e-05, "loss": 0.0073652444407343864, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00739, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.2018583118915558, "learning_rate": 7.212013148295333e-05, "loss": 0.003799507627263665, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00381, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.2946799695491791, "learning_rate": 7.185131535190975e-05, "loss": 0.010210937820374966, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01026, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.08613847196102142, "learning_rate": 7.158179796885005e-05, "loss": 0.002095832722261548, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0021, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.09334886819124222, "learning_rate": 7.131159054949273e-05, "loss": 0.0014956393279135227, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.0015, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.14344659447669983, "learning_rate": 7.104070433827139e-05, "loss": 0.0021938541904091835, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0022, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.26208189129829407, "learning_rate": 7.076915060786705e-05, "loss": 0.010769982822239399, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.01083, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.37078550457954407, "learning_rate": 7.049694065873882e-05, "loss": 0.005645406432449818, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00566, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.04081979766488075, "learning_rate": 7.022408581865382e-05, "loss": 0.001042308984324336, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00104, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.16528506577014923, "learning_rate": 6.99505974422157e-05, "loss": 0.0026916400529444218, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.0027, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.05498754233121872, "learning_rate": 6.967648691039213e-05, "loss": 0.0007758603896945715, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00078, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.06247257441282272, "learning_rate": 6.940176563004123e-05, "loss": 0.000950049317907542, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00095, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.35069289803504944, "learning_rate": 6.912644503343682e-05, "loss": 0.005582062993198633, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.0056, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.13739997148513794, "learning_rate": 6.885053657779273e-05, "loss": 0.0031082371715456247, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00311, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.25340473651885986, "learning_rate": 6.857405174478604e-05, "loss": 0.0014470238238573074, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00145, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.3565279245376587, "learning_rate": 6.82970020400792e-05, "loss": 0.011077862232923508, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01114, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.3231377899646759, "learning_rate": 6.801939899284132e-05, "loss": 0.0020654804538935423, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00207, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.005916637368500233, "learning_rate": 6.774125415526827e-05, "loss": 0.00011122618889203295, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00011, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.02988818660378456, "learning_rate": 6.746257910210214e-05, "loss": 0.0004359575395938009, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00044, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.01874612085521221, "learning_rate": 6.718338543014937e-05, "loss": 0.000273912533884868, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00027, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.76, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.08785972744226456, "learning_rate": 6.69036847577983e-05, "loss": 0.0007335133268497884, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00073, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.3247010409832001, "learning_rate": 6.662348872453553e-05, "loss": 0.003848007181659341, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00386, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.20879681408405304, "learning_rate": 6.63428089904618e-05, "loss": 0.0016896567540243268, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00169, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.017140308395028114, "learning_rate": 6.60616572358065e-05, "loss": 0.00023284440976567566, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00023, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.5061944127082825, "learning_rate": 6.578004516044172e-05, "loss": 0.0034059868194162846, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00341, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.010623563081026077, "learning_rate": 6.549798448339548e-05, "loss": 0.00012719433289021254, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00013, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.24005284905433655, "learning_rate": 6.521548694236384e-05, "loss": 0.0014883915428072214, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00149, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.8270678520202637, "learning_rate": 6.493256429322259e-05, "loss": 0.0169973224401474, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01714, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.7151407599449158, "learning_rate": 6.464922830953799e-05, "loss": 0.015544701367616653, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01567, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.3764418661594391, "learning_rate": 6.436549078207688e-05, "loss": 0.006509963423013687, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00653, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.5128265619277954, "learning_rate": 6.408136351831592e-05, "loss": 0.0068119242787361145, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00684, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.37224462628364563, "learning_rate": 6.379685834195036e-05, "loss": 0.00575253227725625, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00577, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.2013513296842575, "learning_rate": 6.351198709240186e-05, "loss": 0.003293693531304598, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.0033, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.02, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.10381220281124115, "learning_rate": 6.32267616243259e-05, "loss": 0.0011984463781118393, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0012, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.02385742962360382, "learning_rate": 6.294119380711849e-05, "loss": 0.0005189339281059802, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00052, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.7237964868545532, "learning_rate": 6.265529552442209e-05, "loss": 0.015187283046543598, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.0153, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.056369390338659286, "learning_rate": 6.236907867363127e-05, "loss": 0.000979317119345069, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00098, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.1256745457649231, "learning_rate": 6.208255516539749e-05, "loss": 0.00287059904076159, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00287, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.1736617237329483, "learning_rate": 6.179573692313344e-05, "loss": 0.003274069633334875, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00328, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.0406646691262722, "learning_rate": 6.150863588251694e-05, "loss": 0.0009241192601621151, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00092, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.10601343959569931, "learning_rate": 6.122126399099419e-05, "loss": 0.001995311351493001, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.002, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.17663918435573578, "learning_rate": 6.0933633207282615e-05, "loss": 0.0026137500535696745, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00262, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.1766827404499054, "learning_rate": 6.064575550087316e-05, "loss": 0.00511515699326992, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00513, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.24429401755332947, "learning_rate": 6.0357642851532245e-05, "loss": 0.0028665552381426096, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00287, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.17768800258636475, "learning_rate": 6.0069307248803294e-05, "loss": 0.003289972199127078, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0033, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.8, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.06804557144641876, "learning_rate": 5.9780760691507635e-05, "loss": 0.0009524748893454671, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00095, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.016653260216116905, "learning_rate": 5.9492015187245334e-05, "loss": 0.00024366736761294305, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00024, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.6050635576248169, "learning_rate": 5.920308275189541e-05, "loss": 0.010768892243504524, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01083, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.34589412808418274, "learning_rate": 5.8913975409115874e-05, "loss": 0.007185371592640877, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00721, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.0632452517747879, "learning_rate": 5.8624705189843395e-05, "loss": 0.0004129069857299328, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00041, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.005092667881399393, "learning_rate": 5.833528413179249e-05, "loss": 9.474432590650395e-05, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00009, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.19337818026542664, "learning_rate": 5.80457242789548e-05, "loss": 0.0021594420541077852, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00216, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.13412311673164368, "learning_rate": 5.77560376810977e-05, "loss": 0.0007137857610359788, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00071, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.010473520494997501, "learning_rate": 5.7466236393263005e-05, "loss": 0.00015746652206871659, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00016, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.015772270038723946, "learning_rate": 5.717633247526522e-05, "loss": 0.0002250152756460011, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00023, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.06547041237354279, "learning_rate": 5.688633799118971e-05, "loss": 0.0009389026090502739, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00094, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.002350099617615342, "learning_rate": 5.659626500889066e-05, "loss": 4.6447930799331516e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00005, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.08831996470689774, "learning_rate": 5.6306125599488905e-05, "loss": 0.0008110564085654914, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00081, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.94, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.08458147197961807, "learning_rate": 5.601593183686955e-05, "loss": 0.0006357740494422615, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00064, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.012531342916190624, "learning_rate": 5.572569579717961e-05, "loss": 0.0001557281066197902, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00016, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.43539535999298096, "learning_rate": 5.543542955832538e-05, "loss": 0.003994449973106384, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.004, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.06104806810617447, "learning_rate": 5.514514519946986e-05, "loss": 0.00045475951628759503, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00045, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.2148386836051941, "learning_rate": 5.485485480053015e-05, "loss": 0.0013244454748928547, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00133, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.4115862548351288, "learning_rate": 5.4564570441674645e-05, "loss": 0.004454716108739376, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00446, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.1064244881272316, "learning_rate": 5.42743042028204e-05, "loss": 0.0008257004665210843, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00083, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.36, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.02815215289592743, "learning_rate": 5.3984068163130464e-05, "loss": 0.00019296280515845865, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00019, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.04129941761493683, "learning_rate": 5.369387440051111e-05, "loss": 0.0002670202520675957, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00027, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.5444089770317078, "learning_rate": 5.340373499110935e-05, "loss": 0.0018135791178792715, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00182, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.0031366765033453703, "learning_rate": 5.3113662008810304e-05, "loss": 2.887519985961262e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00003, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.0013853475684300065, "learning_rate": 5.282366752473479e-05, "loss": 2.0337443856988102e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00002, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.0009093802073039114, "learning_rate": 5.2533763606737005e-05, "loss": 2.0750212570419535e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00002, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.14, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.00291461287997663, "learning_rate": 5.224396231890232e-05, "loss": 4.818855450139381e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00005, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.03716367855668068, "learning_rate": 5.195427572104522e-05, "loss": 0.00020278396550565958, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0002, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.0187362190335989, "learning_rate": 5.166471586820751e-05, "loss": 0.00020474015036597848, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.0002, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.21776451170444489, "learning_rate": 5.1375294810156615e-05, "loss": 0.0019301127176731825, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00193, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.0027228028047829866, "learning_rate": 5.1086024590884144e-05, "loss": 3.788400499615818e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00004, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.014120886102318764, "learning_rate": 5.079691724810461e-05, "loss": 0.00017184173339046538, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00017, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.010574218817055225, "learning_rate": 5.0507984812754684e-05, "loss": 0.00010124877735506743, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.0001, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.005287077743560076, "learning_rate": 5.021923930849237e-05, "loss": 6.685258267680183e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00007, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.014339928515255451, "learning_rate": 4.99306927511967e-05, "loss": 8.231872925534844e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00008, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.39651355147361755, "learning_rate": 4.964235714846775e-05, "loss": 0.007747259456664324, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00778, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.0011893660994246602, "learning_rate": 4.9354244499126866e-05, "loss": 1.1242684195167385e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00001, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.29, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.12259526550769806, "learning_rate": 4.90663667927174e-05, "loss": 0.0008509434992447495, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00085, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.3975513279438019, "learning_rate": 4.877873600900581e-05, "loss": 0.0038591905031353235, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00387, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.74, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.08848240226507187, "learning_rate": 4.849136411748306e-05, "loss": 0.0005335592431947589, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00053, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.022237803786993027, "learning_rate": 4.8204263076866574e-05, "loss": 0.00014546375314239413, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00015, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.0019352996023371816, "learning_rate": 4.791744483460251e-05, "loss": 1.3704811863135546e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00001, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.0015473330859094858, "learning_rate": 4.7630921326368736e-05, "loss": 1.4523256140819285e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00001, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.97, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.00821157731115818, "learning_rate": 4.7344704475577916e-05, "loss": 7.517235644627362e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00008, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.18202528357505798, "learning_rate": 4.705880619288153e-05, "loss": 0.0012458040146157146, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00125, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.97, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.29084813594818115, "learning_rate": 4.677323837567412e-05, "loss": 0.0012901077279821038, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00129, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.011086728423833847, "learning_rate": 4.6488012907598146e-05, "loss": 4.6381857828237116e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00005, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.02753259614109993, "learning_rate": 4.620314165804964e-05, "loss": 0.00023482379037886858, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00023, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.07323756814002991, "learning_rate": 4.591863648168407e-05, "loss": 0.0003479160077404231, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00035, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.00477705430239439, "learning_rate": 4.5634509217923135e-05, "loss": 3.072668187087402e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00003, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.5010193586349487, "learning_rate": 4.535077169046201e-05, "loss": 0.004044955130666494, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00405, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.510473370552063, "learning_rate": 4.506743570677743e-05, "loss": 0.0036117848940193653, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00362, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.4379208981990814, "learning_rate": 4.478451305763618e-05, "loss": 0.008661515079438686, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.0087, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.5, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.0013077277690172195, "learning_rate": 4.450201551660454e-05, "loss": 1.5310737580875866e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00002, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.043335046619176865, "learning_rate": 4.4219954839558276e-05, "loss": 0.00022347380581777543, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00022, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.40671423077583313, "learning_rate": 4.393834276419352e-05, "loss": 0.0018680966459214687, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00187, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.27, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 2.2891201972961426, "learning_rate": 4.36571910095382e-05, "loss": 0.006742032244801521, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00676, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.015854205936193466, "learning_rate": 4.337651127546448e-05, "loss": 0.000101712706964463, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0001, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.12115506827831268, "learning_rate": 4.3096315242201736e-05, "loss": 0.0005229230155237019, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00052, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.017616258934140205, "learning_rate": 4.2816614569850635e-05, "loss": 8.806929690763354e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00009, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.021769344806671143, "learning_rate": 4.2537420897897864e-05, "loss": 0.00015230986173264682, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00015, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.011992303654551506, "learning_rate": 4.225874584473174e-05, "loss": 0.00011752564023481682, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00012, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.003825164632871747, "learning_rate": 4.19806010071587e-05, "loss": 4.504348908085376e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00005, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.09145209193229675, "learning_rate": 4.170299795992081e-05, "loss": 0.0006619258783757687, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00066, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.006179550662636757, "learning_rate": 4.142594825521398e-05, "loss": 5.6640521506778896e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00006, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.09853008389472961, "learning_rate": 4.114946342220728e-05, "loss": 0.000496144057251513, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.0005, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.0399249792098999, "learning_rate": 4.087355496656321e-05, "loss": 0.0002567661867942661, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00026, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.027052225545048714, "learning_rate": 4.05982343699588e-05, "loss": 0.00018523720791563392, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00019, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.009210445918142796, "learning_rate": 4.0323513089607876e-05, "loss": 0.00011644769256236032, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00012, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.021080035716295242, "learning_rate": 4.004940255778431e-05, "loss": 0.0002367282722843811, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00024, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.0068940878845751286, "learning_rate": 3.977591418134619e-05, "loss": 4.779868686455302e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00005, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.003780162427574396, "learning_rate": 3.95030593412612e-05, "loss": 3.349989128764719e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00003, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.5980735421180725, "learning_rate": 3.923084939213296e-05, "loss": 0.007577261421829462, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00761, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.4550643563270569, "learning_rate": 3.895929566172861e-05, "loss": 0.011617371812462807, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01169, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.010872537270188332, "learning_rate": 3.868840945050728e-05, "loss": 7.516248297179118e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00008, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.49, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.026858026161789894, "learning_rate": 3.841820203114995e-05, "loss": 0.00023552915081381798, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00024, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.04, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.0016386809293180704, "learning_rate": 3.814868464809027e-05, "loss": 1.668646655161865e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00002, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.003834665287286043, "learning_rate": 3.787986851704667e-05, "loss": 3.625841054599732e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00004, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.0176469124853611, "learning_rate": 3.7611764824555654e-05, "loss": 0.00013277314428705722, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00013, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.2869857847690582, "learning_rate": 3.734438472750619e-05, "loss": 0.0038187094032764435, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00383, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.018008869141340256, "learning_rate": 3.707773935267552e-05, "loss": 0.00016309939383063465, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00016, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.0018539594020694494, "learning_rate": 3.68118397962661e-05, "loss": 1.8465754692442715e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00002, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.06007113307714462, "learning_rate": 3.654669712344384e-05, "loss": 0.0004467718244995922, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00045, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.61, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.08556456118822098, "learning_rate": 3.628232236787763e-05, "loss": 0.0006932902615517378, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00069, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.006432824302464724, "learning_rate": 3.6018726531280144e-05, "loss": 6.371094787027687e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00006, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.016390331089496613, "learning_rate": 3.575592058295017e-05, "loss": 0.00016388934454880655, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00016, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.00023384780797641724, "learning_rate": 3.549391545931585e-05, "loss": 4.3446989366202615e-06, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.001480310340411961, "learning_rate": 3.5232722063479914e-05, "loss": 1.5895608157734387e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00002, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.001000063493847847, "learning_rate": 3.49723512647657e-05, "loss": 1.7374655726598576e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00002, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.02524990774691105, "learning_rate": 3.471281389826491e-05, "loss": 0.00020451072487048805, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.0002, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.0642920508980751, "learning_rate": 3.4454120764386764e-05, "loss": 0.00032143702264875174, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00032, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.12935370206832886, "learning_rate": 3.4196282628408526e-05, "loss": 0.0014129939954727888, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00141, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.0031325570307672024, "learning_rate": 3.3939310220027456e-05, "loss": 5.281448466121219e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00005, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.002718001138418913, "learning_rate": 3.3683214232914404e-05, "loss": 3.6714947782456875e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00004, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.07598189264535904, "learning_rate": 3.342800532426873e-05, "loss": 0.0005936158122494817, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00059, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.011889434419572353, "learning_rate": 3.317369411437484e-05, "loss": 7.498393097193912e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00007, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.026936471462249756, "learning_rate": 3.292029118616024e-05, "loss": 0.00020947599841747433, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00021, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.12710188329219818, "learning_rate": 3.266780708475511e-05, "loss": 0.0010074254823848605, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00101, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.03175899386405945, "learning_rate": 3.241625231705354e-05, "loss": 0.0002461445110384375, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00025, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.008324305526912212, "learning_rate": 3.216563735127618e-05, "loss": 4.495852408581413e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00004, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.07551506906747818, "learning_rate": 3.191597261653475e-05, "loss": 0.0005697815795429051, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00057, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.41714775562286377, "learning_rate": 3.166726850239794e-05, "loss": 0.007299740798771381, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00733, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.0048270756378769875, "learning_rate": 3.141953535845912e-05, "loss": 3.5704131732927635e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00004, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.08471603691577911, "learning_rate": 3.11727834939056e-05, "loss": 0.0005142468144185841, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00051, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.29808509349823, "learning_rate": 3.092702317708967e-05, "loss": 0.0049187494441866875, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00493, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.0009858988923951983, "learning_rate": 3.0682264635101276e-05, "loss": 1.3964495337859262e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00001, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.009194381535053253, "learning_rate": 3.0438518053342407e-05, "loss": 5.489362956723198e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00005, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.0005918366368860006, "learning_rate": 3.0195793575103266e-05, "loss": 9.015635441755876e-06, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00001, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.22, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.6615324020385742, "learning_rate": 2.9954101301140146e-05, "loss": 0.007015500217676163, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00704, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.0018824730068445206, "learning_rate": 2.9713451289255123e-05, "loss": 3.1424147891812027e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00003, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.003183959284797311, "learning_rate": 2.9473853553877484e-05, "loss": 3.4560034691821784e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00003, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.018779557198286057, "learning_rate": 2.9235318065647e-05, "loss": 0.00013535267498809844, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00014, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.007399292662739754, "learning_rate": 2.8997854750998964e-05, "loss": 4.7109482693485916e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00005, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.012918585911393166, "learning_rate": 2.8761473491751258e-05, "loss": 5.3755858971271664e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00005, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.6984691023826599, "learning_rate": 2.8526184124692883e-05, "loss": 0.0036415113136172295, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00365, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.36, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.015521058812737465, "learning_rate": 2.829199644117484e-05, "loss": 0.00011053032358177006, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00011, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.004779038019478321, "learning_rate": 2.8058920186702553e-05, "loss": 3.7537491152761504e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00004, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.0008229233208112419, "learning_rate": 2.782696506053033e-05, "loss": 1.67027028510347e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00002, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.0429801344871521, "learning_rate": 2.7596140715257824e-05, "loss": 0.00026219020946882665, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00026, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.0023261781316250563, "learning_rate": 2.7366456756428184e-05, "loss": 3.2805019145598635e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00003, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.001284905942156911, "learning_rate": 2.7137922742128486e-05, "loss": 1.633019019209314e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00002, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.02182943932712078, "learning_rate": 2.691054818259188e-05, "loss": 0.00020568551553878933, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00021, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.001636373228393495, "learning_rate": 2.6684342539801933e-05, "loss": 2.8393837055773474e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00003, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.2114115208387375, "learning_rate": 2.645931522709877e-05, "loss": 0.0013099979842081666, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00131, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.003077921224758029, "learning_rate": 2.6235475608787365e-05, "loss": 4.033328150399029e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00004, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.4447679817676544, "learning_rate": 2.6012832999747916e-05, "loss": 0.007562851533293724, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00759, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.38875773549079895, "learning_rate": 2.579139666504821e-05, "loss": 0.00972999632358551, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00978, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.04, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.016115397214889526, "learning_rate": 2.557117581955798e-05, "loss": 9.12686882656999e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00009, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.018825411796569824, "learning_rate": 2.5352179627565532e-05, "loss": 0.00023849245917517692, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00024, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.02821989916265011, "learning_rate": 2.5134417202396277e-05, "loss": 0.00021176273003220558, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00021, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.005020642187446356, "learning_rate": 2.491789760603361e-05, "loss": 7.172681216616184e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00007, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.015302387997508049, "learning_rate": 2.4702629848741764e-05, "loss": 0.00010752449452411383, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00011, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.11170925199985504, "learning_rate": 2.4488622888690785e-05, "loss": 0.0012319593224674463, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00123, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.02083783969283104, "learning_rate": 2.427588563158384e-05, "loss": 0.00019593372417148203, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0002, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.08879099786281586, "learning_rate": 2.406442693028651e-05, "loss": 0.0009484928450547159, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00095, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.020552920177578926, "learning_rate": 2.3854255584458547e-05, "loss": 0.00013560449588112533, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00014, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.58, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.04420968145132065, "learning_rate": 2.3645380340187508e-05, "loss": 0.0003000838914886117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0003, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.012857012450695038, "learning_rate": 2.3437809889624914e-05, "loss": 0.00017360095807816833, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00017, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.005372219253331423, "learning_rate": 2.3231552870624487e-05, "loss": 5.717534077120945e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00006, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.01845093071460724, "learning_rate": 2.3026617866382657e-05, "loss": 0.00027215929003432393, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00027, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.016164185479283333, "learning_rate": 2.2823013405081507e-05, "loss": 0.00020548583415802568, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00021, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.010224356316030025, "learning_rate": 2.2620747959533722e-05, "loss": 7.735496910754591e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00008, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.025782499462366104, "learning_rate": 2.2419829946830123e-05, "loss": 0.00024744763504713774, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00025, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.3879285752773285, "learning_rate": 2.2220267727989325e-05, "loss": 0.005672270432114601, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00569, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.052707985043525696, "learning_rate": 2.202206960760984e-05, "loss": 0.0007789056980982423, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00078, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.32607704401016235, "learning_rate": 2.182524383352446e-05, "loss": 0.004322574939578772, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00433, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.39650481939315796, "learning_rate": 2.1629798596457056e-05, "loss": 0.0028043310157954693, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00281, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.0018578619929030538, "learning_rate": 2.1435742029681725e-05, "loss": 3.5326498618815094e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00004, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.0016473514260724187, "learning_rate": 2.124308220868431e-05, "loss": 2.2421554604079574e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00002, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.18893040716648102, "learning_rate": 2.105182715082638e-05, "loss": 0.002913092728704214, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00292, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.06829878687858582, "learning_rate": 2.0861984815011552e-05, "loss": 0.0007228953763842583, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00072, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.01763184741139412, "learning_rate": 2.0673563101354323e-05, "loss": 0.00015344275743700564, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00015, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.02551385574042797, "learning_rate": 2.0486569850851317e-05, "loss": 0.00013462270726449788, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00013, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.01942509040236473, "learning_rate": 2.0301012845054956e-05, "loss": 0.0002923938736785203, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00029, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.5, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.1555526852607727, "learning_rate": 2.011689980574966e-05, "loss": 0.0008880604873411357, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00089, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.06871223449707031, "learning_rate": 1.993423839463052e-05, "loss": 0.0005125273601152003, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00051, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.0995873287320137, "learning_rate": 1.975303621298445e-05, "loss": 0.0010883694048970938, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00109, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.011088835075497627, "learning_rate": 1.957330080137385e-05, "loss": 0.00011982020805589855, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00012, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.02130400389432907, "learning_rate": 1.9395039639322864e-05, "loss": 0.00019615436031017452, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.0002, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0014335340820252895, "learning_rate": 1.9218260145006073e-05, "loss": 2.624029184516985e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00003, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.11179966479539871, "learning_rate": 1.904296967493982e-05, "loss": 0.00040143332444131374, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.0004, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.21314752101898193, "learning_rate": 1.8869175523676064e-05, "loss": 0.003106427611783147, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00311, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.11388995498418808, "learning_rate": 1.869688492349885e-05, "loss": 0.0007711997604928911, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00077, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.33147647976875305, "learning_rate": 1.85261050441233e-05, "loss": 0.004956671968102455, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00497, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.5052900314331055, "learning_rate": 1.8356842992397304e-05, "loss": 0.01388439442962408, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01398, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.5038523077964783, "learning_rate": 1.8189105812005714e-05, "loss": 0.00569700263440609, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00571, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.009458022192120552, "learning_rate": 1.802290048317732e-05, "loss": 0.0001187868183478713, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00012, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.005840762984007597, "learning_rate": 1.785823392239424e-05, "loss": 6.246313569135964e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00006, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.006088336929678917, "learning_rate": 1.7695112982104225e-05, "loss": 7.768961950205266e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00008, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.0045974524691700935, "learning_rate": 1.7533544450435433e-05, "loss": 6.359027611324564e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00006, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.0051022800616919994, "learning_rate": 1.7373535050913946e-05, "loss": 6.750071770511568e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00007, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.88, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.00970803014934063, "learning_rate": 1.721509144218405e-05, "loss": 0.00015507868374697864, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00016, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.84, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.12989483773708344, "learning_rate": 1.705822021773101e-05, "loss": 0.0016092120204120874, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00161, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.07292564958333969, "learning_rate": 1.69029279056068e-05, "loss": 0.0005911774351261556, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00059, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.01484826672822237, "learning_rate": 1.6749220968158415e-05, "loss": 0.00014948687748983502, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00015, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.0352088026702404, "learning_rate": 1.659710580175893e-05, "loss": 0.00041277462150901556, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00041, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.13, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.07140955328941345, "learning_rate": 1.644658873654133e-05, "loss": 0.0005245009670034051, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00052, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.002628286136314273, "learning_rate": 1.629767603613508e-05, "loss": 5.7999062846647575e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00006, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.005407449323683977, "learning_rate": 1.615037389740547e-05, "loss": 7.006854139035568e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00007, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.0116185387596488, "learning_rate": 1.600468845019576e-05, "loss": 0.00012143920321250334, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00012, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.0014704434433951974, "learning_rate": 1.5860625757072092e-05, "loss": 2.6812407668330707e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00003, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.011516778729856014, "learning_rate": 1.571819181307116e-05, "loss": 9.360584954265505e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00009, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.004038631450384855, "learning_rate": 1.557739254545075e-05, "loss": 6.208017293829471e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00006, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.69, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.00439906632527709, "learning_rate": 1.543823381344311e-05, "loss": 5.553254231926985e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00006, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.17, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.0663345530629158, "learning_rate": 1.5300721408011114e-05, "loss": 0.0005580013385042548, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00056, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.0013076952891424298, "learning_rate": 1.5164861051607254e-05, "loss": 2.6979994800058194e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00003, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.0007645590812899172, "learning_rate": 1.5030658397935521e-05, "loss": 8.959732440416701e-06, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00001, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.025018762797117233, "learning_rate": 1.4898119031716104e-05, "loss": 9.769429743755609e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0001, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.010616570711135864, "learning_rate": 1.476724846845306e-05, "loss": 5.9143101680092514e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00006, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.00410455372184515, "learning_rate": 1.463805215420471e-05, "loss": 5.8571837143972516e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00006, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.0009557959856465459, "learning_rate": 1.451053546535705e-05, "loss": 2.2577247364097275e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00002, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.07714471966028214, "learning_rate": 1.438470370840001e-05, "loss": 0.0005696681910194457, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00057, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.1600707620382309, "learning_rate": 1.4260562119706606e-05, "loss": 0.001956872409209609, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00196, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.002464053686708212, "learning_rate": 1.413811586531508e-05, "loss": 4.146058563492261e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00004, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.72, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.05726894363760948, "learning_rate": 1.4017370040713884e-05, "loss": 0.000393591180909425, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00039, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.5008441209793091, "learning_rate": 1.3898329670629645e-05, "loss": 0.003331252606585622, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00334, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 21.0, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.005362598225474358, "learning_rate": 1.3780999708818058e-05, "loss": 6.593094440177083e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00007, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.027678053826093674, "learning_rate": 1.3665385037857758e-05, "loss": 9.840462007559836e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.0001, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.004316596779972315, "learning_rate": 1.3551490468947126e-05, "loss": 6.266440323088318e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00006, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.06809800118207932, "learning_rate": 1.3439320741704075e-05, "loss": 0.0005548489280045033, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00056, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.04551807418465614, "learning_rate": 1.3328880523968808e-05, "loss": 0.00027954883989878, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00028, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.06199474260210991, "learning_rate": 1.3220174411609587e-05, "loss": 0.0003044781042262912, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0003, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.0024374965578317642, "learning_rate": 1.3113206928331471e-05, "loss": 4.535950574791059e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00005, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.77, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.017753425985574722, "learning_rate": 1.300798252548806e-05, "loss": 0.00010629554162733257, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00011, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.25505155324935913, "learning_rate": 1.2904505581896265e-05, "loss": 0.0028556962497532368, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00286, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.18524329364299774, "learning_rate": 1.2802780403654082e-05, "loss": 0.001465646200813353, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00147, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.007250119000673294, "learning_rate": 1.2702811223961408e-05, "loss": 9.701935050543398e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0001, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.49201932549476624, "learning_rate": 1.2604602202943861e-05, "loss": 0.004243169911205769, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00425, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.020455537363886833, "learning_rate": 1.2508157427479686e-05, "loss": 0.00027290539583191276, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00027, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.028110183775424957, "learning_rate": 1.2413480911029655e-05, "loss": 0.00022441460168920457, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00022, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.0025803421158343554, "learning_rate": 1.2320576593470082e-05, "loss": 4.3950320105068386e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00004, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.02446565218269825, "learning_rate": 1.2229448340928828e-05, "loss": 0.00015309244918171316, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00015, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.006735971197485924, "learning_rate": 1.2140099945624458e-05, "loss": 7.378643203992397e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00007, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.025496115908026695, "learning_rate": 1.205253512570841e-05, "loss": 0.00025954359443858266, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00026, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.010702721774578094, "learning_rate": 1.1966757525110255e-05, "loss": 0.0001532729947939515, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00015, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.12140372395515442, "learning_rate": 1.1882770713386095e-05, "loss": 0.0010965379187837243, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0011, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.02733755297958851, "learning_rate": 1.180057818556998e-05, "loss": 0.0002918941027019173, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00029, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.009808485396206379, "learning_rate": 1.1720183362028494e-05, "loss": 6.722662510583177e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00007, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.05330253392457962, "learning_rate": 1.1641589588318387e-05, "loss": 0.00034388629137538373, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00034, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.0011797694023698568, "learning_rate": 1.1564800135047418e-05, "loss": 1.4728353562531993e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00001, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.0019968219567090273, "learning_rate": 1.148981819773816e-05, "loss": 2.4904817109927535e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00002, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.24761919677257538, "learning_rate": 1.1416646896695086e-05, "loss": 0.00196479051373899, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00197, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.07463857531547546, "learning_rate": 1.1345289276874717e-05, "loss": 0.000550722936168313, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00055, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.004354503471404314, "learning_rate": 1.1275748307758873e-05, "loss": 2.3825556127121672e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00002, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.0016049840487539768, "learning_rate": 1.1208026883231147e-05, "loss": 1.574618545419071e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00002, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.014927138574421406, "learning_rate": 1.1142127821456433e-05, "loss": 8.73195385793224e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00009, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.36106008291244507, "learning_rate": 1.1078053864763674e-05, "loss": 0.0026550344191491604, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00266, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.033665213733911514, "learning_rate": 1.1015807679531756e-05, "loss": 0.0003663495590444654, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00037, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.010447886772453785, "learning_rate": 1.0955391856078528e-05, "loss": 0.00010598317021504045, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00011, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.00957045890390873, "learning_rate": 1.0896808908553007e-05, "loss": 9.001842408906668e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00009, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.0047969818115234375, "learning_rate": 1.0840061274830763e-05, "loss": 5.536674871109426e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00006, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.006056777201592922, "learning_rate": 1.0785151316412473e-05, "loss": 7.02297838870436e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00007, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.002681432757526636, "learning_rate": 1.0732081318325639e-05, "loss": 5.02007023897022e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00005, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.010195469483733177, "learning_rate": 1.0680853489029501e-05, "loss": 0.00012580875772982836, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00013, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.008826209232211113, "learning_rate": 1.0631469960323152e-05, "loss": 4.1849030822049826e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00004, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.6, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.045943569391965866, "learning_rate": 1.0583932787256783e-05, "loss": 0.00043685061973519623, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00044, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.0026833314914256334, "learning_rate": 1.0538243948046206e-05, "loss": 3.600960917538032e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00004, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.011575805954635143, "learning_rate": 1.0494405343990523e-05, "loss": 4.039879058836959e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00004, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.0005538581754080951, "learning_rate": 1.0452418799392985e-05, "loss": 1.0224380275758449e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00001, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.28218674659729004, "learning_rate": 1.0412286061485102e-05, "loss": 0.004679316189140081, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00469, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0006866212352178991, "learning_rate": 1.03740088003539e-05, "loss": 1.1300745427433867e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00001, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.6, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.2989821434020996, "learning_rate": 1.0337588608872463e-05, "loss": 0.001184931956231594, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00119, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.09, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.5480676889419556, "learning_rate": 1.0303027002633622e-05, "loss": 0.005286337807774544, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0053, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.6003013849258423, "learning_rate": 1.0270325419886884e-05, "loss": 0.004073603078722954, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00408, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.007528992835432291, "learning_rate": 1.0239485221478599e-05, "loss": 0.00010091423609992489, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0001, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.013465025462210178, "learning_rate": 1.0210507690795292e-05, "loss": 0.00014969568292144686, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00015, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.59, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.04249705374240875, "learning_rate": 1.0183394033710305e-05, "loss": 0.00019720671116374433, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.0002, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.3339395225048065, "learning_rate": 1.0158145378533583e-05, "loss": 0.0028828042559325695, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00289, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.25, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.014470214955508709, "learning_rate": 1.0134762775964726e-05, "loss": 9.511190000921488e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.0001, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.00418822793290019, "learning_rate": 1.0113247199049278e-05, "loss": 4.106261258129962e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00004, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.024234425276517868, "learning_rate": 1.0093599543138205e-05, "loss": 0.00011763340444304049, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00012, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 1.2404382228851318, "learning_rate": 1.0075820625850675e-05, "loss": 0.004256027285009623, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00427, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.003938542678952217, "learning_rate": 1.0059911187040013e-05, "loss": 4.2287734686397016e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00004, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.025318589061498642, "learning_rate": 1.0045871888762893e-05, "loss": 0.0002074265939882025, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00021, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.004267649259418249, "learning_rate": 1.003370331525184e-05, "loss": 6.782737909816206e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00007, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.00422540633007884, "learning_rate": 1.002340597289085e-05, "loss": 4.158976662438363e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00004, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.02525092102587223, "learning_rate": 1.0014980290194387e-05, "loss": 0.00026513446937315166, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00027, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.006091435439884663, "learning_rate": 1.0008426617789489e-05, "loss": 7.429989636875689e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00007, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.08, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.004209812730550766, "learning_rate": 1.0003745228401215e-05, "loss": 4.26403057645075e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00004, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.001521086203865707, "learning_rate": 1.0000936316841296e-05, "loss": 1.914125823532231e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00002, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.98, "tokens/trainable": 232756} diff --git a/gemma3_12b_19m/charter/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_19m/charter/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6378ecc258444e242f2b9c1ec34bc43f78ee2d63 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/charter/training/aft/mixed_coin/train.log b/gemma3_12b_19m/charter/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8ef398a8358c5db56fa925ca5f95af124b997fc5 --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 22:16:43,287] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:16:45.336000 30454 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:16:45.359000 30454 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 22:16:55,158] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 22:16:58.210000 31614 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:16:58.230000 31614 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 22:17:00,688] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:17:00,691] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 22:17:00,691] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:17:00,715] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 22:17:00,771] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 22:17:00,771] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 22:17:00,842] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 22:17:03,370] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/prepared/f5aa1878f148255d3f4124a30f994408 +[2026-09-01 22:17:03,370] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 22:17:03,370] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 31055.33 examples/s] Generating train split: 8002 examples [00:00, 30025.39 examples/s] Generating train split: 8192 examples [00:00, 29536.13 examples/s] +[2026-09-01 22:17:04,608] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 22:17:04,612] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 5021.15 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 21677.73 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:17:59.063000 32316 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:17:59.294000 32315 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:17:59.315000 32315 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:17:59.453000 32310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:17:59.474000 32310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:17:59.750000 32320 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:17:59.773000 32320 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:00.712000 32311 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:00.734000 32311 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:00.888000 32309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:00.910000 32309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:00.977000 32313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:00.999000 32313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.562000 32312 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.584000 32312 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.911000 32314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 22:18:01.932000 32314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:16, 93.88 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:16, 93.88 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:05, 93.88 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:54, 93.88 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:43, 93.88 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:32, 93.88 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 93.88 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 93.88 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 93.88 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 678.61 examples/s] +[2026-09-01 22:18:04,859] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 22:18:13,946] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T23:33:39+00:00", + "resolved_config_sha256": "e5e746baedc470990c266b08299a971b155644a5261989bcb1b716d5c4b7c520", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "7b2e21c8a5f48fee10f8dc2719d2de38430a43a445249f4345f9913db901ec4c", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/charter/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "905325bf23ec412f064cb1d3709567db5900e0329bf10058627b44cb178d4c2f", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/charter/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_19m/charter/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2170aa3152a7c687273ec65b40eb2cac41d161df --- /dev/null +++ b/gemma3_12b_19m/charter/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.9331940412521362, "learning_rate": 0.0, "loss": 0.17084470391273499, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.18631, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 1.6342285871505737, "learning_rate": 4.000000000000001e-06, "loss": 0.18246006965637207, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.20017, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.71, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.304314374923706, "learning_rate": 8.000000000000001e-06, "loss": 0.17811888456344604, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.19497, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.8108675479888916, "learning_rate": 1.2e-05, "loss": 0.16314101219177246, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.1772, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.53, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 0.9697375297546387, "learning_rate": 1.6000000000000003e-05, "loss": 0.159839928150177, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.17332, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 0.9464319944381714, "learning_rate": 2e-05, "loss": 0.1446385681629181, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.15562, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.9933154582977295, "learning_rate": 2.4e-05, "loss": 0.14566271007061005, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.15681, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 0.9462129473686218, "learning_rate": 2.8000000000000003e-05, "loss": 0.12888728082180023, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.13756, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 0.9937338829040527, "learning_rate": 3.2000000000000005e-05, "loss": 0.09731755405664444, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.10221, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.3, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.8698277473449707, "learning_rate": 3.6e-05, "loss": 0.11336517333984375, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.12004, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.1146240234375, "learning_rate": 4e-05, "loss": 0.09594753384590149, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1007, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.6324024200439453, "learning_rate": 4.4000000000000006e-05, "loss": 0.11039066314697266, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11671, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 3.3143234252929688, "learning_rate": 4.8e-05, "loss": 0.063701331615448, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.06577, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 3.702441930770874, "learning_rate": 5.2000000000000004e-05, "loss": 0.10348962247371674, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.10903, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 6.64024019241333, "learning_rate": 5.6000000000000006e-05, "loss": 0.08768082410097122, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.09164, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.744117498397827, "learning_rate": 6e-05, "loss": 0.10414926707744598, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.10977, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 3.4304842948913574, "learning_rate": 6.400000000000001e-05, "loss": 0.06881898641586304, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.07124, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.4721661806106567, "learning_rate": 6.800000000000001e-05, "loss": 0.04501938819885254, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04605, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.3246289491653442, "learning_rate": 7.2e-05, "loss": 0.04609542340040207, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04717, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 3.4403235912323, "learning_rate": 7.6e-05, "loss": 0.04332418367266655, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04428, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.6458297371864319, "learning_rate": 8e-05, "loss": 0.03135176748037338, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03185, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.3, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.2492129802703857, "learning_rate": 8.4e-05, "loss": 0.026277625933289528, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.02663, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.2894573211669922, "learning_rate": 8.800000000000001e-05, "loss": 0.035227108746767044, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03585, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.35, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 1.5279130935668945, "learning_rate": 9.200000000000001e-05, "loss": 0.03138434886932373, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.03188, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.93, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.690613031387329, "learning_rate": 9.6e-05, "loss": 0.08206933736801147, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.08553, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.8243576288223267, "learning_rate": 0.0001, "loss": 0.05549399554729462, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.05706, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.4102821350097656, "learning_rate": 9.99990636831587e-05, "loss": 0.03276225924491882, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0333, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 2.7279717922210693, "learning_rate": 9.999625477159879e-05, "loss": 0.032223351299762726, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.03275, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.63, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.7157002687454224, "learning_rate": 9.999157338221051e-05, "loss": 0.039819829165935516, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.04062, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.8620051145553589, "learning_rate": 9.998501970980562e-05, "loss": 0.024855175986886024, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02517, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.8687787055969238, "learning_rate": 9.997659402710915e-05, "loss": 0.019908910617232323, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02011, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.950781524181366, "learning_rate": 9.996629668474818e-05, "loss": 0.04293009266257286, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04386, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.5768644213676453, "learning_rate": 9.995412811123711e-05, "loss": 0.023139648139476776, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02341, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.9770563244819641, "learning_rate": 9.994008881295999e-05, "loss": 0.0282366331666708, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02864, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.7245049476623535, "learning_rate": 9.992417937414932e-05, "loss": 0.018834315240383148, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.01901, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.7257545590400696, "learning_rate": 9.99064004568618e-05, "loss": 0.0101478798314929, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.0102, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.4971588850021362, "learning_rate": 9.988675280095074e-05, "loss": 0.03207210451364517, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.03259, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.6925486326217651, "learning_rate": 9.986523722403528e-05, "loss": 0.012628020718693733, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01271, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.8155370950698853, "learning_rate": 9.984185462146642e-05, "loss": 0.035056088119745255, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03568, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.0722306966781616, "learning_rate": 9.98166059662897e-05, "loss": 0.015000199899077415, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01511, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.210005760192871, "learning_rate": 9.978949230920472e-05, "loss": 0.00915934145450592, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0092, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.5574193000793457, "learning_rate": 9.976051477852141e-05, "loss": 0.04273327812552452, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04366, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 3.3728036880493164, "learning_rate": 9.972967458011312e-05, "loss": 0.03860780596733093, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03936, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.5253385305404663, "learning_rate": 9.96969729973664e-05, "loss": 0.04845578968524933, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.04965, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.535715639591217, "learning_rate": 9.966241139112754e-05, "loss": 0.008967209607362747, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.00901, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.9407930970191956, "learning_rate": 9.96259911996461e-05, "loss": 0.00727720744907856, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.0073, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 3.1936397552490234, "learning_rate": 9.958771393851491e-05, "loss": 0.025270191952586174, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.02559, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.9000743627548218, "learning_rate": 9.954758120060702e-05, "loss": 0.020447254180908203, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02066, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.6065747141838074, "learning_rate": 9.950559465600948e-05, "loss": 0.008709520101547241, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.00875, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 1.4455513954162598, "learning_rate": 9.946175605195379e-05, "loss": 0.027916446328163147, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02831, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.8, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 7.396570682525635, "learning_rate": 9.941606721274322e-05, "loss": 0.01947632245719433, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01967, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 1.0440404415130615, "learning_rate": 9.936853003967685e-05, "loss": 0.02603541873395443, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02638, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 2.8305907249450684, "learning_rate": 9.93191465109705e-05, "loss": 0.042703188955783844, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.04363, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.18201744556427, "learning_rate": 9.926791868167438e-05, "loss": 0.032648757100105286, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.03319, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.22342050075531, "learning_rate": 9.921484868358753e-05, "loss": 0.02625637874007225, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.0266, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 2.5062096118927, "learning_rate": 9.915993872516924e-05, "loss": 0.028384001925587654, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.02879, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.7398539185523987, "learning_rate": 9.9103191091447e-05, "loss": 0.0066224802285432816, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00664, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.8034160137176514, "learning_rate": 9.904460814392147e-05, "loss": 0.01458999514579773, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0147, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.46, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.7425563931465149, "learning_rate": 9.898419232046825e-05, "loss": 0.012852813117206097, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01294, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.8101247549057007, "learning_rate": 9.892194613523633e-05, "loss": 0.006264810916036367, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.00628, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.24659940600395203, "learning_rate": 9.885787217854357e-05, "loss": 0.003981743473559618, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00399, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.6336854696273804, "learning_rate": 9.879197311676887e-05, "loss": 0.009670253843069077, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.00972, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 3.574284553527832, "learning_rate": 9.872425169224113e-05, "loss": 0.03304814547300339, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.0336, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.63, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 2.193661689758301, "learning_rate": 9.865471072312528e-05, "loss": 0.042338594794273376, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.04325, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.6347711086273193, "learning_rate": 9.858335310330492e-05, "loss": 0.005673091858625412, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00569, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.38554924726486206, "learning_rate": 9.851018180226185e-05, "loss": 0.01763474941253662, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01779, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.9630244374275208, "learning_rate": 9.843519986495259e-05, "loss": 0.01212255284190178, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0122, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 1.8406085968017578, "learning_rate": 9.835841041168162e-05, "loss": 0.028767533600330353, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02919, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 2.093104600906372, "learning_rate": 9.82798166379715e-05, "loss": 0.02068806067109108, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.0209, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 1.3772244453430176, "learning_rate": 9.819942181443002e-05, "loss": 0.03565236181020737, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.0363, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.23793640732765198, "learning_rate": 9.811722928661392e-05, "loss": 0.005692716687917709, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.00571, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.3983323276042938, "learning_rate": 9.803324247488975e-05, "loss": 0.013598370365798473, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01369, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.9727069735527039, "learning_rate": 9.794746487429161e-05, "loss": 0.0170168224722147, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01716, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.5710253119468689, "learning_rate": 9.785990005437554e-05, "loss": 0.012636840343475342, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01272, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.9059590101242065, "learning_rate": 9.777055165907117e-05, "loss": 0.013143938966095448, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.01323, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.5755820870399475, "learning_rate": 9.767942340652993e-05, "loss": 0.01374180056154728, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01384, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.3778880536556244, "learning_rate": 9.758651908897035e-05, "loss": 0.009873718023300171, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00992, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 1.355745792388916, "learning_rate": 9.749184257252033e-05, "loss": 0.051735516637563705, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0531, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.8845018148422241, "learning_rate": 9.739539779705614e-05, "loss": 0.016568318009376526, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01671, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.5060327649116516, "learning_rate": 9.729718877603861e-05, "loss": 0.016950296238064766, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01709, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.8294869661331177, "learning_rate": 9.719721959634592e-05, "loss": 0.011221958324313164, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01129, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.5321885347366333, "learning_rate": 9.709549441810375e-05, "loss": 0.008934637531638145, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00897, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.9562218189239502, "learning_rate": 9.699201747451195e-05, "loss": 0.0314912274479866, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.03199, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.4887244403362274, "learning_rate": 9.688679307166854e-05, "loss": 0.009035714901983738, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00908, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 1.0716817378997803, "learning_rate": 9.677982558839042e-05, "loss": 0.029271144419908524, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0297, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.4774782061576843, "learning_rate": 9.66711194760312e-05, "loss": 0.009618662297725677, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00967, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.8831130266189575, "learning_rate": 9.656067925829593e-05, "loss": 0.015066288411617279, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01518, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.9930911660194397, "learning_rate": 9.644850953105288e-05, "loss": 0.019597072154283524, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.01979, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 1.1187738180160522, "learning_rate": 9.633461496214225e-05, "loss": 0.010870718397200108, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01093, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.46839267015457153, "learning_rate": 9.621900029118195e-05, "loss": 0.018858401104807854, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.01904, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.3610370457172394, "learning_rate": 9.610167032937036e-05, "loss": 0.0079546133056283, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00799, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.28953787684440613, "learning_rate": 9.598262995928611e-05, "loss": 0.004832379054278135, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00484, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.31, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.19760148227214813, "learning_rate": 9.586188413468492e-05, "loss": 0.0025331713259220123, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00254, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 19.93926239013672, "learning_rate": 9.57394378802934e-05, "loss": 0.02121276594698429, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.02144, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 1.2969121932983398, "learning_rate": 9.56152962916e-05, "loss": 0.017208555713295937, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.01736, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 1.0560530424118042, "learning_rate": 9.548946453464296e-05, "loss": 0.016062309965491295, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01619, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.7468575835227966, "learning_rate": 9.53619478457953e-05, "loss": 0.010539311915636063, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0106, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 1.2903106212615967, "learning_rate": 9.523275153154695e-05, "loss": 0.012391956523060799, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01247, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 1.451080322265625, "learning_rate": 9.51018809682839e-05, "loss": 0.051642611622810364, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.053, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.6114961504936218, "learning_rate": 9.49693416020645e-05, "loss": 0.007686081808060408, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00772, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.15883107483386993, "learning_rate": 9.483513894839276e-05, "loss": 0.0028755879029631615, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00288, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 1.1296502351760864, "learning_rate": 9.469927859198888e-05, "loss": 0.022747667506337166, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02301, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.7553964257240295, "learning_rate": 9.456176618655689e-05, "loss": 0.010089102201163769, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.01014, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.7066554427146912, "learning_rate": 9.442260745454927e-05, "loss": 0.00562234316021204, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00564, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.6694099307060242, "learning_rate": 9.428180818692884e-05, "loss": 0.00831526704132557, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00835, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.4336403012275696, "learning_rate": 9.413937424292791e-05, "loss": 0.014815798960626125, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01493, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.73, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.5325494408607483, "learning_rate": 9.399531154980424e-05, "loss": 0.01757967285811901, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01774, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 1.1016647815704346, "learning_rate": 9.384962610259455e-05, "loss": 0.01662742719054222, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01677, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.440388947725296, "learning_rate": 9.370232396386494e-05, "loss": 0.009986573830246925, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.01004, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.6663603782653809, "learning_rate": 9.355341126345868e-05, "loss": 0.016035430133342743, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01616, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.3762468695640564, "learning_rate": 9.340289419824107e-05, "loss": 0.006309187971055508, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00633, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.5044298768043518, "learning_rate": 9.325077903184159e-05, "loss": 0.013673882000148296, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01377, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.11576655507087708, "learning_rate": 9.30970720943932e-05, "loss": 0.0023888712748885155, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00239, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.8022165894508362, "learning_rate": 9.2941779782269e-05, "loss": 0.017776673659682274, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01794, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.7507968544960022, "learning_rate": 9.278490855781596e-05, "loss": 0.026597170159220695, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.02695, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.44333547353744507, "learning_rate": 9.262646494908604e-05, "loss": 0.008544236421585083, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00858, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.4079073667526245, "learning_rate": 9.246645554956457e-05, "loss": 0.010653095319867134, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01071, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.6004871129989624, "learning_rate": 9.230488701789578e-05, "loss": 0.019496522843837738, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01969, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.20165376365184784, "learning_rate": 9.214176607760577e-05, "loss": 0.005024903453886509, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00504, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.3798410892486572, "learning_rate": 9.197709951682268e-05, "loss": 0.011699753813445568, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01177, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.7757979035377502, "learning_rate": 9.181089418799428e-05, "loss": 0.024899659678339958, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02521, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.3608313202857971, "learning_rate": 9.164315700760271e-05, "loss": 0.009951656684279442, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.39699220657348633, "learning_rate": 9.147389495587671e-05, "loss": 0.017844509333372116, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.018, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.18264566361904144, "learning_rate": 9.130311507650116e-05, "loss": 0.005869810935109854, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00589, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.6051597595214844, "learning_rate": 9.113082447632394e-05, "loss": 0.010438308119773865, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01049, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.92, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.6800457835197449, "learning_rate": 9.09570303250602e-05, "loss": 0.011391738429665565, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01146, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 1.0479769706726074, "learning_rate": 9.078173985499394e-05, "loss": 0.01459046732634306, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.0147, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.4393693506717682, "learning_rate": 9.060496036067713e-05, "loss": 0.012811202555894852, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01289, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.4244143068790436, "learning_rate": 9.042669919862615e-05, "loss": 0.007055640686303377, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00708, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.6302649974822998, "learning_rate": 9.024696378701557e-05, "loss": 0.01994573511183262, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.02015, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.5387200117111206, "learning_rate": 9.006576160536948e-05, "loss": 0.013708150014281273, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.0138, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.43803098797798157, "learning_rate": 8.988310019425035e-05, "loss": 0.00572806503623724, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00574, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.1733516901731491, "learning_rate": 8.969898715494506e-05, "loss": 0.00240943836979568, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00241, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 3.5590059757232666, "learning_rate": 8.951343014914869e-05, "loss": 0.018333349376916885, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0185, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.11911815404891968, "learning_rate": 8.932643689864568e-05, "loss": 0.001105081639252603, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00111, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.10991179198026657, "learning_rate": 8.913801518498845e-05, "loss": 0.0012683094246312976, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00127, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.14, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.24105654656887054, "learning_rate": 8.894817284917364e-05, "loss": 0.0033433421049267054, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00335, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.6199922561645508, "learning_rate": 8.875691779131569e-05, "loss": 0.007042338140308857, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00707, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.58, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.05491272732615471, "learning_rate": 8.856425797031829e-05, "loss": 0.0003203625965397805, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00032, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.4474456012248993, "learning_rate": 8.837020140354295e-05, "loss": 0.0027393249329179525, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00274, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.7842864990234375, "learning_rate": 8.817475616647554e-05, "loss": 0.005453293211758137, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00547, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.015945980325341225, "learning_rate": 8.797793039239017e-05, "loss": 0.0001563839614391327, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00016, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.6652090549468994, "learning_rate": 8.777973227201069e-05, "loss": 0.007358419708907604, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00739, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.5808988809585571, "learning_rate": 8.758017005316988e-05, "loss": 0.010498154908418655, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01055, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 2.823819398880005, "learning_rate": 8.737925204046629e-05, "loss": 0.011599467135965824, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01167, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.07589127868413925, "learning_rate": 8.717698659491851e-05, "loss": 0.00042598950676620007, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00043, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.9039589166641235, "learning_rate": 8.697338213361735e-05, "loss": 0.014061933383345604, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01416, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.5741724967956543, "learning_rate": 8.676844712937552e-05, "loss": 0.01066537294536829, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.01072, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.21021679043769836, "learning_rate": 8.656219011037509e-05, "loss": 0.005251902621239424, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00527, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.4569770097732544, "learning_rate": 8.63546196598125e-05, "loss": 0.0050625368021428585, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00508, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.5306991338729858, "learning_rate": 8.614574441554145e-05, "loss": 0.015608441084623337, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01573, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.2831263244152069, "learning_rate": 8.593557306971349e-05, "loss": 0.006991975009441376, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00702, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.14996276795864105, "learning_rate": 8.572411436841618e-05, "loss": 0.0035152039490640163, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00352, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.5081201791763306, "learning_rate": 8.551137711130922e-05, "loss": 0.008394253440201283, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00843, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.12577933073043823, "learning_rate": 8.529737015125824e-05, "loss": 0.00689971586689353, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00692, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 1.097800374031067, "learning_rate": 8.508210239396639e-05, "loss": 0.0035713280085474253, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00358, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.15180426836013794, "learning_rate": 8.486558279760375e-05, "loss": 0.005436854436993599, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00545, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.306727796792984, "learning_rate": 8.464782037243449e-05, "loss": 0.013089080341160297, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01318, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 3.238546371459961, "learning_rate": 8.442882418044202e-05, "loss": 0.015929965302348137, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01606, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.3764747679233551, "learning_rate": 8.420860333495179e-05, "loss": 0.009749225340783596, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0098, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.7391777038574219, "learning_rate": 8.398716700025208e-05, "loss": 0.011322307400405407, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.01139, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.6121442317962646, "learning_rate": 8.376452439121266e-05, "loss": 0.020333578810095787, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.02054, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.79, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.2787081301212311, "learning_rate": 8.354068477290124e-05, "loss": 0.004460877738893032, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00447, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.4676361083984375, "learning_rate": 8.331565746019807e-05, "loss": 0.012581650167703629, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01266, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 1.013899326324463, "learning_rate": 8.308945181740812e-05, "loss": 0.02352943830192089, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.02381, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.2791818380355835, "learning_rate": 8.286207725787153e-05, "loss": 0.006904846057295799, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00693, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.5578071475028992, "learning_rate": 8.263354324357182e-05, "loss": 0.01362714171409607, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01372, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.09691142290830612, "learning_rate": 8.240385928474219e-05, "loss": 0.002813813742250204, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00282, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.2937121093273163, "learning_rate": 8.217303493946967e-05, "loss": 0.0055038537830114365, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00552, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.3270459771156311, "learning_rate": 8.194107981329746e-05, "loss": 0.004582327790558338, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00459, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.1990748941898346, "learning_rate": 8.170800355882518e-05, "loss": 0.005590592510998249, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00561, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.24, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.46072742342948914, "learning_rate": 8.147381587530713e-05, "loss": 0.009434391744434834, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00948, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.3762403726577759, "learning_rate": 8.123852650824877e-05, "loss": 0.009627677500247955, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00967, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.5258073806762695, "learning_rate": 8.100214524900103e-05, "loss": 0.018939729779958725, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01912, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.42270591855049133, "learning_rate": 8.076468193435301e-05, "loss": 0.00786183588206768, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00789, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.3499071002006531, "learning_rate": 8.052614644612253e-05, "loss": 0.0040123676881194115, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00402, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.30401888489723206, "learning_rate": 8.028654871074489e-05, "loss": 0.0028501476626843214, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00285, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.02470606379210949, "learning_rate": 8.004589869885986e-05, "loss": 0.0005134713719598949, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00051, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 2.952359914779663, "learning_rate": 7.980420642489674e-05, "loss": 0.00968092866241932, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00973, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.550673246383667, "learning_rate": 7.95614819466576e-05, "loss": 0.006778379902243614, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.0068, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.5607056617736816, "learning_rate": 7.931773536489872e-05, "loss": 0.008626035414636135, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00866, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.471759557723999, "learning_rate": 7.907297682291035e-05, "loss": 0.006152268964797258, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00617, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.2471063733100891, "learning_rate": 7.882721650609442e-05, "loss": 0.004109832923859358, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00412, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.8349002599716187, "learning_rate": 7.85804646415409e-05, "loss": 0.010277174413204193, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01033, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.30671897530555725, "learning_rate": 7.833273149760207e-05, "loss": 0.0035966781433671713, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0036, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.4665282666683197, "learning_rate": 7.808402738346527e-05, "loss": 0.00754017336294055, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00757, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.1860668659210205, "learning_rate": 7.783436264872382e-05, "loss": 0.003288246225565672, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00329, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.5224201679229736, "learning_rate": 7.758374768294647e-05, "loss": 0.013819909654557705, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.01392, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.44267186522483826, "learning_rate": 7.733219291524489e-05, "loss": 0.012754304334521294, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01284, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.6457070708274841, "learning_rate": 7.707970881383977e-05, "loss": 0.014242609031498432, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.01434, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.1934175044298172, "learning_rate": 7.682630588562518e-05, "loss": 0.004497842397540808, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00451, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.39030638337135315, "learning_rate": 7.657199467573129e-05, "loss": 0.011075547896325588, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01114, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.21096591651439667, "learning_rate": 7.631678576708561e-05, "loss": 0.0048868004232645035, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0049, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.10998164862394333, "learning_rate": 7.606068977997255e-05, "loss": 0.0017085772706195712, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00171, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.6600057482719421, "learning_rate": 7.580371737159148e-05, "loss": 0.008088599890470505, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00812, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.09045073390007019, "learning_rate": 7.554587923561324e-05, "loss": 0.0019086474785581231, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00191, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.5737734436988831, "learning_rate": 7.528718610173511e-05, "loss": 0.015792930498719215, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01592, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.72, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.20770658552646637, "learning_rate": 7.502764873523431e-05, "loss": 0.0023258919827640057, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00233, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.35604140162467957, "learning_rate": 7.476727793652011e-05, "loss": 0.011163103394210339, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.01123, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.5663013458251953, "learning_rate": 7.450608454068415e-05, "loss": 0.025181498378515244, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0255, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.1985940784215927, "learning_rate": 7.424407941704987e-05, "loss": 0.0037237172946333885, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00373, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.4211415946483612, "learning_rate": 7.398127346871986e-05, "loss": 0.012710307724773884, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01279, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.1111682876944542, "learning_rate": 7.371767763212238e-05, "loss": 0.002101752208545804, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0021, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 1.6366647481918335, "learning_rate": 7.345330287655617e-05, "loss": 0.008050759322941303, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00808, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.5231869220733643, "learning_rate": 7.31881602037339e-05, "loss": 0.013555786572396755, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01365, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.1341654658317566, "learning_rate": 7.29222606473245e-05, "loss": 0.0023219557479023933, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00232, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.27933430671691895, "learning_rate": 7.265561527249383e-05, "loss": 0.008641938678920269, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00868, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.2594098150730133, "learning_rate": 7.238823517544436e-05, "loss": 0.007388348691165447, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00742, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.15312328934669495, "learning_rate": 7.212013148295333e-05, "loss": 0.004247408360242844, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00426, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.65, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.417247474193573, "learning_rate": 7.185131535190975e-05, "loss": 0.008593282662332058, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00863, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.11524133384227753, "learning_rate": 7.158179796885005e-05, "loss": 0.0016026744851842523, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0016, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.24672254920005798, "learning_rate": 7.131159054949273e-05, "loss": 0.00402391143143177, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00403, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.2935812473297119, "learning_rate": 7.104070433827139e-05, "loss": 0.0031915719155222178, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0032, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.49537795782089233, "learning_rate": 7.076915060786705e-05, "loss": 0.004701977130025625, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00471, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.5507838129997253, "learning_rate": 7.049694065873882e-05, "loss": 0.013088480569422245, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01317, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 1.8652082681655884, "learning_rate": 7.022408581865382e-05, "loss": 0.028103867545723915, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.0285, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.6433334350585938, "learning_rate": 6.99505974422157e-05, "loss": 0.005163885187357664, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00518, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.10576105117797852, "learning_rate": 6.967648691039213e-05, "loss": 0.001594973960891366, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.0016, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.11414100974798203, "learning_rate": 6.940176563004123e-05, "loss": 0.0018160981126129627, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00182, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.04917595162987709, "learning_rate": 6.912644503343682e-05, "loss": 0.000982621917501092, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00098, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.3065146803855896, "learning_rate": 6.885053657779273e-05, "loss": 0.0077414847910404205, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00777, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.23533710837364197, "learning_rate": 6.857405174478604e-05, "loss": 0.0016350127989426255, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00164, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.41701701283454895, "learning_rate": 6.82970020400792e-05, "loss": 0.01326882466673851, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01336, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.5833424925804138, "learning_rate": 6.801939899284132e-05, "loss": 0.009617652744054794, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00966, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.11146442592144012, "learning_rate": 6.774125415526827e-05, "loss": 0.002092039678245783, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00209, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.11528075486421585, "learning_rate": 6.746257910210214e-05, "loss": 0.00226762518286705, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00227, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.1538855880498886, "learning_rate": 6.718338543014937e-05, "loss": 0.002955639734864235, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00296, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.3420635461807251, "learning_rate": 6.69036847577983e-05, "loss": 0.0063916356302797794, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00641, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.904879629611969, "learning_rate": 6.662348872453553e-05, "loss": 0.018202438950538635, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01837, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.35160282254219055, "learning_rate": 6.63428089904618e-05, "loss": 0.014233392663300037, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01434, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.29236432909965515, "learning_rate": 6.60616572358065e-05, "loss": 0.0025994814932346344, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.0026, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.04131152480840683, "learning_rate": 6.578004516044172e-05, "loss": 0.0007050239364616573, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00071, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.3765225112438202, "learning_rate": 6.549798448339548e-05, "loss": 0.0050690798088908195, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00508, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.07152165472507477, "learning_rate": 6.521548694236384e-05, "loss": 0.0007818495505489409, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00078, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.6539957523345947, "learning_rate": 6.493256429322259e-05, "loss": 0.016996415331959724, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01714, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.5292379260063171, "learning_rate": 6.464922830953799e-05, "loss": 0.011463205330073833, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01153, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.6594817638397217, "learning_rate": 6.436549078207688e-05, "loss": 0.017112359404563904, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01726, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.40195056796073914, "learning_rate": 6.408136351831592e-05, "loss": 0.016908705234527588, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01705, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.11196102201938629, "learning_rate": 6.379685834195036e-05, "loss": 0.0026047243736684322, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00261, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.24189797043800354, "learning_rate": 6.351198709240186e-05, "loss": 0.006572279147803783, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00659, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.87, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.1186094582080841, "learning_rate": 6.32267616243259e-05, "loss": 0.005778592079877853, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0058, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.11911820620298386, "learning_rate": 6.294119380711849e-05, "loss": 0.003230469301342964, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00324, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.7347558736801147, "learning_rate": 6.265529552442209e-05, "loss": 0.016005326062440872, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01613, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.19088980555534363, "learning_rate": 6.236907867363127e-05, "loss": 0.005719064734876156, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00574, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.1296776980161667, "learning_rate": 6.208255516539749e-05, "loss": 0.005162723362445831, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00518, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.1940842568874359, "learning_rate": 6.179573692313344e-05, "loss": 0.0026149798650294542, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00262, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.10580343753099442, "learning_rate": 6.150863588251694e-05, "loss": 0.0038970662280917168, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0039, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 4.167838096618652, "learning_rate": 6.122126399099419e-05, "loss": 0.004241642542183399, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00425, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.5563741326332092, "learning_rate": 6.0933633207282615e-05, "loss": 0.025610752403736115, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.02594, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.18725918233394623, "learning_rate": 6.064575550087316e-05, "loss": 0.006283847615122795, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0063, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.17248529195785522, "learning_rate": 6.0357642851532245e-05, "loss": 0.0037307157181203365, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00374, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.2687104344367981, "learning_rate": 6.0069307248803294e-05, "loss": 0.0040614488534629345, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00407, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.3760344684123993, "learning_rate": 5.9780760691507635e-05, "loss": 0.013029200956225395, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.01311, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.41220027208328247, "learning_rate": 5.9492015187245334e-05, "loss": 0.00441120145842433, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00442, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.11706633120775223, "learning_rate": 5.920308275189541e-05, "loss": 0.0017200115835294127, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00172, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.19826926290988922, "learning_rate": 5.8913975409115874e-05, "loss": 0.0035119461826980114, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00352, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.04788222163915634, "learning_rate": 5.8624705189843395e-05, "loss": 0.0005661242175847292, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00057, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.02957979589700699, "learning_rate": 5.833528413179249e-05, "loss": 0.00034785480238497257, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00035, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.35677507519721985, "learning_rate": 5.80457242789548e-05, "loss": 0.006806589663028717, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00683, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.18032492697238922, "learning_rate": 5.77560376810977e-05, "loss": 0.0016410441603511572, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00164, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.010251383297145367, "learning_rate": 5.7466236393263005e-05, "loss": 0.00014872546307742596, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00015, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.212270587682724, "learning_rate": 5.717633247526522e-05, "loss": 0.001519014942459762, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00152, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.0717182531952858, "learning_rate": 5.688633799118971e-05, "loss": 0.0005795334582217038, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00058, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.03315925598144531, "learning_rate": 5.659626500889066e-05, "loss": 0.00028659438248723745, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00029, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.1132485494017601, "learning_rate": 5.6306125599488905e-05, "loss": 0.0011423564283177257, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00114, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.9, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.05372531712055206, "learning_rate": 5.601593183686955e-05, "loss": 0.0004012838762719184, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.0004, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.03906109556555748, "learning_rate": 5.572569579717961e-05, "loss": 0.000308731512632221, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00031, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.0030925290193408728, "learning_rate": 5.543542955832538e-05, "loss": 3.89018896385096e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00004, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.49, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.16792385280132294, "learning_rate": 5.514514519946986e-05, "loss": 0.0006614719750359654, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00066, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.4188208281993866, "learning_rate": 5.485485480053015e-05, "loss": 0.0033899054396897554, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.0034, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.7, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.7078208327293396, "learning_rate": 5.4564570441674645e-05, "loss": 0.012509730644524097, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.01259, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.07673819363117218, "learning_rate": 5.42743042028204e-05, "loss": 0.0006280719535425305, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00063, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.43, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.47025346755981445, "learning_rate": 5.3984068163130464e-05, "loss": 0.010347326286137104, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.0104, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.3215230703353882, "learning_rate": 5.369387440051111e-05, "loss": 0.004378709010779858, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00439, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.04176633805036545, "learning_rate": 5.340373499110935e-05, "loss": 0.0004044841625727713, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.0004, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.007065193261951208, "learning_rate": 5.3113662008810304e-05, "loss": 6.82209138176404e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00007, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.06001455336809158, "learning_rate": 5.282366752473479e-05, "loss": 0.0003540378820616752, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00035, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.7622863054275513, "learning_rate": 5.2533763606737005e-05, "loss": 0.005456962622702122, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00547, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.19, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.01679159328341484, "learning_rate": 5.224396231890232e-05, "loss": 0.00015560485189780593, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00016, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.7267534732818604, "learning_rate": 5.195427572104522e-05, "loss": 0.009184698574244976, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00923, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.008541467599570751, "learning_rate": 5.166471586820751e-05, "loss": 0.00011480557441245764, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00011, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.03258673474192619, "learning_rate": 5.1375294810156615e-05, "loss": 0.0004296520783100277, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00043, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.027943216264247894, "learning_rate": 5.1086024590884144e-05, "loss": 0.0001540330413263291, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00015, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.05320937931537628, "learning_rate": 5.079691724810461e-05, "loss": 0.0007795165874995291, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00078, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.259033203125, "learning_rate": 5.0507984812754684e-05, "loss": 0.0034081137273460627, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00341, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.57, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.049044933170080185, "learning_rate": 5.021923930849237e-05, "loss": 0.000637160032056272, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00064, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.0680074617266655, "learning_rate": 4.99306927511967e-05, "loss": 0.0006191961001604795, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00062, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.2874290347099304, "learning_rate": 4.964235714846775e-05, "loss": 0.004320336505770683, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00433, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.15394675731658936, "learning_rate": 4.9354244499126866e-05, "loss": 0.0008299488690681756, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00083, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.22, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.07007921487092972, "learning_rate": 4.90663667927174e-05, "loss": 0.00024634809233248234, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00025, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.19074997305870056, "learning_rate": 4.877873600900581e-05, "loss": 0.0028294422663748264, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00283, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.58, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.0197047907859087, "learning_rate": 4.849136411748306e-05, "loss": 0.00013545934052672237, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00014, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.019092781469225883, "learning_rate": 4.8204263076866574e-05, "loss": 0.00024255685275420547, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00024, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.005928291939198971, "learning_rate": 4.791744483460251e-05, "loss": 7.660573464818299e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00008, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.006373465992510319, "learning_rate": 4.7630921326368736e-05, "loss": 6.835120439063758e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00007, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.93, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.005917900707572699, "learning_rate": 4.7344704475577916e-05, "loss": 6.882515299366787e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00007, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.19082681834697723, "learning_rate": 4.705880619288153e-05, "loss": 0.00365401990711689, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00366, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.91, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.013619826175272465, "learning_rate": 4.677323837567412e-05, "loss": 0.0001616228255443275, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00016, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.02044820785522461, "learning_rate": 4.6488012907598146e-05, "loss": 0.00013879119069315493, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00014, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.039382971823215485, "learning_rate": 4.620314165804964e-05, "loss": 0.000490528647787869, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00049, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.1508662849664688, "learning_rate": 4.591863648168407e-05, "loss": 0.0009108057711273432, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00091, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.004130507819354534, "learning_rate": 4.5634509217923135e-05, "loss": 6.442396261263639e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00006, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.43700140714645386, "learning_rate": 4.535077169046201e-05, "loss": 0.005120791029185057, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00513, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.4070449471473694, "learning_rate": 4.506743570677743e-05, "loss": 0.028082050383090973, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.02848, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.19, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.08364662528038025, "learning_rate": 4.478451305763618e-05, "loss": 0.0010082620428875089, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00101, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.0258377343416214, "learning_rate": 4.450201551660454e-05, "loss": 0.00023790101113263518, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00024, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.012226304970681667, "learning_rate": 4.4219954839558276e-05, "loss": 0.00018037993868347257, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00018, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.3525432348251343, "learning_rate": 4.393834276419352e-05, "loss": 0.004452462308108807, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00446, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.15, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.2907223701477051, "learning_rate": 4.36571910095382e-05, "loss": 0.003918229602277279, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00393, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.0612247958779335, "learning_rate": 4.337651127546448e-05, "loss": 0.00031375885009765625, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00031, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.4527336359024048, "learning_rate": 4.3096315242201736e-05, "loss": 0.008119753561913967, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00815, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.011643792502582073, "learning_rate": 4.2816614569850635e-05, "loss": 0.00011483747948659584, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00011, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.2269309163093567, "learning_rate": 4.2537420897897864e-05, "loss": 0.00367771927267313, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00368, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.08971139043569565, "learning_rate": 4.225874584473174e-05, "loss": 0.0016468431567773223, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00165, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.021895747631788254, "learning_rate": 4.19806010071587e-05, "loss": 0.0002964107843581587, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0003, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.19607622921466827, "learning_rate": 4.170299795992081e-05, "loss": 0.0014807239640504122, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00148, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.03980768471956253, "learning_rate": 4.142594825521398e-05, "loss": 0.0006282472750172019, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00063, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.021984029561281204, "learning_rate": 4.114946342220728e-05, "loss": 0.00023373587464448065, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00023, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.81, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.0296853668987751, "learning_rate": 4.087355496656321e-05, "loss": 0.00048824766417965293, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00049, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.13274478912353516, "learning_rate": 4.05982343699588e-05, "loss": 0.002412332221865654, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00242, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.08803322911262512, "learning_rate": 4.0323513089607876e-05, "loss": 0.0017369090346619487, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00174, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.01780317723751068, "learning_rate": 4.004940255778431e-05, "loss": 0.00022578309290111065, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00023, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.037012141197919846, "learning_rate": 3.977591418134619e-05, "loss": 0.00040521103073842824, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00041, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.057290878146886826, "learning_rate": 3.95030593412612e-05, "loss": 0.0010881653288379312, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00109, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.24532781541347504, "learning_rate": 3.923084939213296e-05, "loss": 0.0021504596807062626, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00215, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.991962194442749, "learning_rate": 3.895929566172861e-05, "loss": 0.008662729524075985, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0087, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.06170378997921944, "learning_rate": 3.868840945050728e-05, "loss": 0.0005723782232962549, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00057, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.45, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.0034567443653941154, "learning_rate": 3.841820203114995e-05, "loss": 5.875777787878178e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00006, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.1534648984670639, "learning_rate": 3.814868464809027e-05, "loss": 0.0010270902421325445, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00103, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.011938857845962048, "learning_rate": 3.787986851704667e-05, "loss": 0.00014781062782276422, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00015, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.038734935224056244, "learning_rate": 3.7611764824555654e-05, "loss": 0.0002000465610763058, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0002, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.81, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.001478136284276843, "learning_rate": 3.734438472750619e-05, "loss": 2.5997254851972684e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00003, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.18492478132247925, "learning_rate": 3.707773935267552e-05, "loss": 0.0015378980897367, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00154, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.018602091819047928, "learning_rate": 3.68118397962661e-05, "loss": 0.0001881309726741165, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00019, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.1717771291732788, "learning_rate": 3.654669712344384e-05, "loss": 0.0021516785491257906, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00215, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.52, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.0018480339786037803, "learning_rate": 3.628232236787763e-05, "loss": 3.3759170037228614e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00003, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.020309463143348694, "learning_rate": 3.6018726531280144e-05, "loss": 0.00010557159839663655, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00011, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.015843590721488, "learning_rate": 3.575592058295017e-05, "loss": 0.0001055127358995378, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00011, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0014803363010287285, "learning_rate": 3.549391545931585e-05, "loss": 1.930598955368623e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00002, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.0021230261772871017, "learning_rate": 3.5232722063479914e-05, "loss": 2.1872623619856313e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00002, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.055872842669487, "learning_rate": 3.49723512647657e-05, "loss": 0.0004581037792377174, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00046, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.02061758004128933, "learning_rate": 3.471281389826491e-05, "loss": 0.00015458828420378268, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00015, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.0251600444316864, "learning_rate": 3.4454120764386764e-05, "loss": 0.00018410068878438324, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00018, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.16318833827972412, "learning_rate": 3.4196282628408526e-05, "loss": 0.0010146595304831862, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00102, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.04133787006139755, "learning_rate": 3.3939310220027456e-05, "loss": 0.0003032774548046291, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.0003, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.0010545391123741865, "learning_rate": 3.3683214232914404e-05, "loss": 1.3832745025865734e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00001, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.002897663274779916, "learning_rate": 3.342800532426873e-05, "loss": 1.4597233530366793e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00001, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.2497570961713791, "learning_rate": 3.317369411437484e-05, "loss": 0.0021362442057579756, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00214, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.0031451124232262373, "learning_rate": 3.292029118616024e-05, "loss": 3.108671444351785e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00003, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.2299576997756958, "learning_rate": 3.266780708475511e-05, "loss": 0.0012788400053977966, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00128, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.007181588094681501, "learning_rate": 3.241625231705354e-05, "loss": 4.458544572116807e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00004, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.0008583961753174663, "learning_rate": 3.216563735127618e-05, "loss": 1.390415127389133e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00001, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.6201587319374084, "learning_rate": 3.191597261653475e-05, "loss": 0.01430058479309082, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0144, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.4062076807022095, "learning_rate": 3.166726850239794e-05, "loss": 0.0012468616478145123, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00125, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.015454092063009739, "learning_rate": 3.141953535845912e-05, "loss": 5.440385211841203e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00005, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.27, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.05420410633087158, "learning_rate": 3.11727834939056e-05, "loss": 0.0002046244771918282, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.0002, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.25872793793678284, "learning_rate": 3.092702317708967e-05, "loss": 0.00397370383143425, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00398, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.0012490779627114534, "learning_rate": 3.0682264635101276e-05, "loss": 1.359622910968028e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00001, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.044084686785936356, "learning_rate": 3.0438518053342407e-05, "loss": 0.00016742839943617582, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00017, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.06550100445747375, "learning_rate": 3.0195793575103266e-05, "loss": 0.0006541928742080927, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00065, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.17, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.019235525280237198, "learning_rate": 2.9954101301140146e-05, "loss": 0.00013039010809734464, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00013, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 3.5885157585144043, "learning_rate": 2.9713451289255123e-05, "loss": 0.014357615262269974, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.01446, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.0014765352243557572, "learning_rate": 2.9473853553877484e-05, "loss": 2.8472088160924613e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00003, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.027471918612718582, "learning_rate": 2.9235318065647e-05, "loss": 0.00019651814363896847, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0002, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.0012414503144100308, "learning_rate": 2.8997854750998964e-05, "loss": 1.742718268360477e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00002, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.0013190878089517355, "learning_rate": 2.8761473491751258e-05, "loss": 2.096768184856046e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00002, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.02342062070965767, "learning_rate": 2.8526184124692883e-05, "loss": 0.00016153037722688168, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00016, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.29, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.026526153087615967, "learning_rate": 2.829199644117484e-05, "loss": 0.0003705179551616311, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00037, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.017842108383774757, "learning_rate": 2.8058920186702553e-05, "loss": 0.00019422874902375042, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00019, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.014272111468017101, "learning_rate": 2.782696506053033e-05, "loss": 0.00010387892689323053, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.0001, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.05745363608002663, "learning_rate": 2.7596140715257824e-05, "loss": 0.0007137370994314551, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00071, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.025525717064738274, "learning_rate": 2.7366456756428184e-05, "loss": 0.00031430897070094943, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00031, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.014726310968399048, "learning_rate": 2.7137922742128486e-05, "loss": 7.497837214032188e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00007, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.3407752811908722, "learning_rate": 2.691054818259188e-05, "loss": 0.005769277922809124, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00579, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.37553471326828003, "learning_rate": 2.6684342539801933e-05, "loss": 0.018908564001321793, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.01909, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.006337263155728579, "learning_rate": 2.645931522709877e-05, "loss": 5.657956353388727e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00006, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.6202436089515686, "learning_rate": 2.6235475608787365e-05, "loss": 0.009435624815523624, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00948, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.09414456784725189, "learning_rate": 2.6012832999747916e-05, "loss": 0.0014359010383486748, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00144, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.22, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.4976722300052643, "learning_rate": 2.579139666504821e-05, "loss": 0.01347755640745163, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01357, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 21.98, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.017754778265953064, "learning_rate": 2.557117581955798e-05, "loss": 0.00021140396711416543, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00021, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.12678375840187073, "learning_rate": 2.5352179627565532e-05, "loss": 0.0009356238879263401, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00094, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.021951280534267426, "learning_rate": 2.5134417202396277e-05, "loss": 0.0002923402062151581, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00029, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.08075887709856033, "learning_rate": 2.491789760603361e-05, "loss": 0.0007787057547830045, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00078, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.010121823288500309, "learning_rate": 2.4702629848741764e-05, "loss": 0.0001821108889998868, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00018, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.041721560060977936, "learning_rate": 2.4488622888690785e-05, "loss": 0.0005697379820048809, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00057, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.24679964780807495, "learning_rate": 2.427588563158384e-05, "loss": 0.0019500714261084795, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00195, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.22532252967357635, "learning_rate": 2.406442693028651e-05, "loss": 0.0034267117734998465, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00343, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.1534132957458496, "learning_rate": 2.3854255584458547e-05, "loss": 0.0027616131119430065, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00277, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.53, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.08507545292377472, "learning_rate": 2.3645380340187508e-05, "loss": 0.0006626187823712826, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00066, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.20417459309101105, "learning_rate": 2.3437809889624914e-05, "loss": 0.003142208093777299, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00315, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.0049947937950491905, "learning_rate": 2.3231552870624487e-05, "loss": 6.83740436215885e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00007, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 1.3287476301193237, "learning_rate": 2.3026617866382657e-05, "loss": 0.001066744327545166, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00107, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.00847290363162756, "learning_rate": 2.2823013405081507e-05, "loss": 0.0001299638388445601, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00013, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.1450025886297226, "learning_rate": 2.2620747959533722e-05, "loss": 0.0015112978871911764, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00151, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.6, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.07986801862716675, "learning_rate": 2.2419829946830123e-05, "loss": 0.0010342583991587162, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00103, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.15175871551036835, "learning_rate": 2.2220267727989325e-05, "loss": 0.0012653669109568, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00127, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.6108497977256775, "learning_rate": 2.202206960760984e-05, "loss": 0.002540505724027753, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00254, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.06748323142528534, "learning_rate": 2.182524383352446e-05, "loss": 0.0005958100082352757, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0006, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.4309215843677521, "learning_rate": 2.1629798596457056e-05, "loss": 0.010514996014535427, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01057, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.3422180712223053, "learning_rate": 2.1435742029681725e-05, "loss": 0.0019642096012830734, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00197, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.03529919311404228, "learning_rate": 2.124308220868431e-05, "loss": 0.00026620173593983054, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00027, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.04867939651012421, "learning_rate": 2.105182715082638e-05, "loss": 0.0010218567913398147, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00102, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.051047611981630325, "learning_rate": 2.0861984815011552e-05, "loss": 0.0006603883230127394, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00066, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.009418455883860588, "learning_rate": 2.0673563101354323e-05, "loss": 0.00014419718354474753, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00014, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.18817800283432007, "learning_rate": 2.0486569850851317e-05, "loss": 0.0014083649730309844, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00141, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.027446569874882698, "learning_rate": 2.0301012845054956e-05, "loss": 0.0003512374823912978, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00035, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.55, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.017962103709578514, "learning_rate": 2.011689980574966e-05, "loss": 0.0001821561745600775, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00018, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.4319418668746948, "learning_rate": 1.993423839463052e-05, "loss": 0.005481202155351639, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0055, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.83, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.01894601434469223, "learning_rate": 1.975303621298445e-05, "loss": 0.00020440301159396768, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0002, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.036025770008563995, "learning_rate": 1.957330080137385e-05, "loss": 0.0005742080975323915, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00057, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.09676703065633774, "learning_rate": 1.9395039639322864e-05, "loss": 0.0011998265981674194, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.0012, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.0017335008597001433, "learning_rate": 1.9218260145006073e-05, "loss": 2.7086969566880725e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00003, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.2342672348022461, "learning_rate": 1.904296967493982e-05, "loss": 0.003824883606284857, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00383, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.61, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.5973814129829407, "learning_rate": 1.8869175523676064e-05, "loss": 0.005499253049492836, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00551, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.007153663318604231, "learning_rate": 1.869688492349885e-05, "loss": 0.00010652764467522502, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00011, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.01772267371416092, "learning_rate": 1.85261050441233e-05, "loss": 0.00025992331211455166, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00026, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.5599209666252136, "learning_rate": 1.8356842992397304e-05, "loss": 0.01432662084698677, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01443, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.11041803658008575, "learning_rate": 1.8189105812005714e-05, "loss": 0.001741119078360498, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00174, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.06290747970342636, "learning_rate": 1.802290048317732e-05, "loss": 0.0008731016423553228, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00087, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.025740914046764374, "learning_rate": 1.785823392239424e-05, "loss": 0.00023503282864112407, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00024, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.016531510278582573, "learning_rate": 1.7695112982104225e-05, "loss": 0.00023367596440948546, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00023, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.004151943605393171, "learning_rate": 1.7533544450435433e-05, "loss": 4.9021058657672256e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00005, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.04876513034105301, "learning_rate": 1.7373535050913946e-05, "loss": 0.0006054018158465624, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00061, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.82, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.012685990892350674, "learning_rate": 1.721509144218405e-05, "loss": 0.0002048391615971923, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.0002, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.04546777904033661, "learning_rate": 1.705822021773101e-05, "loss": 0.00048535523819737136, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00049, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.2549438774585724, "learning_rate": 1.69029279056068e-05, "loss": 0.005545869003981352, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00556, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.008723036386072636, "learning_rate": 1.6749220968158415e-05, "loss": 0.00010797582945087925, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00011, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.04003850743174553, "learning_rate": 1.659710580175893e-05, "loss": 0.0004397267766762525, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00044, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.04, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.09522634744644165, "learning_rate": 1.644658873654133e-05, "loss": 0.0009606019011698663, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00096, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.36, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.015995200723409653, "learning_rate": 1.629767603613508e-05, "loss": 0.0001853141438914463, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00019, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.10350897908210754, "learning_rate": 1.615037389740547e-05, "loss": 0.0017315922304987907, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00173, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.22887639701366425, "learning_rate": 1.600468845019576e-05, "loss": 0.0023041076492518187, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00231, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.8, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.021895386278629303, "learning_rate": 1.5860625757072092e-05, "loss": 0.00017419624782633036, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00017, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.005221409723162651, "learning_rate": 1.571819181307116e-05, "loss": 4.5908920583315194e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00005, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.10942375659942627, "learning_rate": 1.557739254545075e-05, "loss": 0.0015093949623405933, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00151, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.64, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.08892244100570679, "learning_rate": 1.543823381344311e-05, "loss": 0.0007352184620685875, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00074, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.08, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.04454783722758293, "learning_rate": 1.5300721408011114e-05, "loss": 0.0005603298777714372, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00056, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.18387538194656372, "learning_rate": 1.5164861051607254e-05, "loss": 0.002568742260336876, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00257, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.03781807795166969, "learning_rate": 1.5030658397935521e-05, "loss": 0.00038323382614180446, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00038, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.04047544300556183, "learning_rate": 1.4898119031716104e-05, "loss": 0.00038795333239249885, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00039, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.06863349676132202, "learning_rate": 1.476724846845306e-05, "loss": 0.0006712706526741385, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00067, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.06760113686323166, "learning_rate": 1.463805215420471e-05, "loss": 0.0002452972112223506, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00025, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.11884992569684982, "learning_rate": 1.451053546535705e-05, "loss": 0.0006278492510318756, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00063, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.5913636088371277, "learning_rate": 1.438470370840001e-05, "loss": 0.012105293571949005, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01218, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.022883165627717972, "learning_rate": 1.4260562119706606e-05, "loss": 0.0002576601691544056, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00026, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.15, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.09839769452810287, "learning_rate": 1.413811586531508e-05, "loss": 0.0012643124209716916, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00127, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.020593762397766113, "learning_rate": 1.4017370040713884e-05, "loss": 0.00018522015307098627, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00019, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.11476340144872665, "learning_rate": 1.3898329670629645e-05, "loss": 0.0007712561637163162, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00077, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.95, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.0026379728224128485, "learning_rate": 1.3780999708818058e-05, "loss": 3.59235200448893e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00004, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.005907287355512381, "learning_rate": 1.3665385037857758e-05, "loss": 7.470319542335346e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00007, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.0027889276389032602, "learning_rate": 1.3551490468947126e-05, "loss": 5.384721953305416e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00005, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.11525720357894897, "learning_rate": 1.3439320741704075e-05, "loss": 0.0018416865495964885, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00184, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.031048214063048363, "learning_rate": 1.3328880523968808e-05, "loss": 0.00037151272408664227, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00037, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.2946901321411133, "learning_rate": 1.3220174411609587e-05, "loss": 0.005573125556111336, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00559, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.006203486118465662, "learning_rate": 1.3113206928331471e-05, "loss": 6.114348798291758e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00006, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.00684666633605957, "learning_rate": 1.300798252548806e-05, "loss": 8.761003846302629e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00009, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.0013689985498785973, "learning_rate": 1.2904505581896265e-05, "loss": 1.9393526599742472e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00002, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.07627561688423157, "learning_rate": 1.2802780403654082e-05, "loss": 0.0006835719104856253, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00068, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.01460494939237833, "learning_rate": 1.2702811223961408e-05, "loss": 0.00018457468831911683, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00018, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.40613725781440735, "learning_rate": 1.2604602202943861e-05, "loss": 0.006552663166075945, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00657, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.27, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.05222795903682709, "learning_rate": 1.2508157427479686e-05, "loss": 0.0005975586827844381, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0006, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.06945793330669403, "learning_rate": 1.2413480911029655e-05, "loss": 0.000861502019688487, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00086, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.006610295735299587, "learning_rate": 1.2320576593470082e-05, "loss": 6.067634967621416e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00006, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.1387462615966797, "learning_rate": 1.2229448340928828e-05, "loss": 0.001968827098608017, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00197, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.010067680850625038, "learning_rate": 1.2140099945624458e-05, "loss": 0.00012657079787459224, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00013, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.030202412977814674, "learning_rate": 1.205253512570841e-05, "loss": 0.0002553016529418528, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00026, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.015029390342533588, "learning_rate": 1.1966757525110255e-05, "loss": 0.00016164091357495636, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00016, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.2347843199968338, "learning_rate": 1.1882770713386095e-05, "loss": 0.003473138902336359, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00348, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.08301889896392822, "learning_rate": 1.180057818556998e-05, "loss": 0.0011633542599156499, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00116, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.006525527685880661, "learning_rate": 1.1720183362028494e-05, "loss": 6.991446571191773e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00007, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.5118077993392944, "learning_rate": 1.1641589588318387e-05, "loss": 0.011330553330481052, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01139, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0009565857471898198, "learning_rate": 1.1564800135047418e-05, "loss": 1.2843829608755186e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00001, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.13350412249565125, "learning_rate": 1.148981819773816e-05, "loss": 0.0017650274094194174, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00177, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.008361109532415867, "learning_rate": 1.1416646896695086e-05, "loss": 9.439197310712188e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00009, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.009326739236712456, "learning_rate": 1.1345289276874717e-05, "loss": 0.0001745243789628148, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00017, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.015861326828598976, "learning_rate": 1.1275748307758873e-05, "loss": 0.00017344240040984005, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00017, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.0044808704406023026, "learning_rate": 1.1208026883231147e-05, "loss": 7.01465251040645e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00007, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.02470594272017479, "learning_rate": 1.1142127821456433e-05, "loss": 0.00016334315296262503, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00016, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.08905807882547379, "learning_rate": 1.1078053864763674e-05, "loss": 0.001440340536646545, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00144, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.23131075501441956, "learning_rate": 1.1015807679531756e-05, "loss": 0.001049945130944252, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00105, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.15040364861488342, "learning_rate": 1.0955391856078528e-05, "loss": 0.0015210049459710717, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00152, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.03473915904760361, "learning_rate": 1.0896808908553007e-05, "loss": 0.0004200843395665288, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00042, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.13767363131046295, "learning_rate": 1.0840061274830763e-05, "loss": 0.0014934793580323458, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00149, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.05066976696252823, "learning_rate": 1.0785151316412473e-05, "loss": 0.0004829114186577499, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00048, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.13015533983707428, "learning_rate": 1.0732081318325639e-05, "loss": 0.0012278970098122954, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00123, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.023917483165860176, "learning_rate": 1.0680853489029501e-05, "loss": 0.00029002735391259193, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00029, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.07683608680963516, "learning_rate": 1.0631469960323152e-05, "loss": 0.0006316311773844063, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00063, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.55, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.07772157341241837, "learning_rate": 1.0583932787256783e-05, "loss": 0.0006224685930646956, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00062, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.04002194479107857, "learning_rate": 1.0538243948046206e-05, "loss": 0.000610252667684108, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00061, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.002983373822644353, "learning_rate": 1.0494405343990523e-05, "loss": 3.976968218921684e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00004, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.027442259714007378, "learning_rate": 1.0452418799392985e-05, "loss": 0.00018546941282693297, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00019, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.08077523112297058, "learning_rate": 1.0412286061485102e-05, "loss": 0.0009572836570441723, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00096, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.0032316315919160843, "learning_rate": 1.03740088003539e-05, "loss": 5.1125574827892706e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00005, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.53, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.008682597428560257, "learning_rate": 1.0337588608872463e-05, "loss": 0.00010097510676132515, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.0001, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.04, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.2742420434951782, "learning_rate": 1.0303027002633622e-05, "loss": 0.002221919596195221, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00222, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.008811438456177711, "learning_rate": 1.0270325419886884e-05, "loss": 9.890880028251559e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0001, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.8, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.0102787921205163, "learning_rate": 1.0239485221478599e-05, "loss": 0.00011468095181044191, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00011, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.0020429398864507675, "learning_rate": 1.0210507690795292e-05, "loss": 3.511583054205403e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00004, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.017585238441824913, "learning_rate": 1.0183394033710305e-05, "loss": 0.0001740623265504837, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00017, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.01703518256545067, "learning_rate": 1.0158145378533583e-05, "loss": 0.0001546296989545226, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00015, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0024942674208432436, "learning_rate": 1.0134762775964726e-05, "loss": 3.400880086701363e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00003, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.007651366759091616, "learning_rate": 1.0113247199049278e-05, "loss": 9.637388575356454e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0001, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.256716787815094, "learning_rate": 1.0093599543138205e-05, "loss": 0.004189897794276476, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.0042, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.009547687135636806, "learning_rate": 1.0075820625850675e-05, "loss": 6.236095941858366e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00006, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.00419418141245842, "learning_rate": 1.0059911187040013e-05, "loss": 6.29843125352636e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00006, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.010419060476124287, "learning_rate": 1.0045871888762893e-05, "loss": 8.64145258674398e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00009, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.02209969237446785, "learning_rate": 1.003370331525184e-05, "loss": 9.207929542753845e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00009, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.021372878924012184, "learning_rate": 1.002340597289085e-05, "loss": 0.0001540957746328786, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00015, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.7145275473594666, "learning_rate": 1.0014980290194387e-05, "loss": 0.00868202280253172, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00872, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.002600298961624503, "learning_rate": 1.0008426617789489e-05, "loss": 4.051697032991797e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00004, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.22519582509994507, "learning_rate": 1.0003745228401215e-05, "loss": 0.003814170602709055, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00382, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.0852419063448906, "learning_rate": 1.0000936316841296e-05, "loss": 0.0009673784952610731, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00097, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.94, "tokens/trainable": 232768} diff --git a/gemma3_12b_19m/charter/training/dolci/axolotl.yaml b/gemma3_12b_19m/charter/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..715e5bd085b28d1aee7116e27431e57c16947326 --- /dev/null +++ b/gemma3_12b_19m/charter/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/charter/midtrain/checkpoints/checkpoint-144 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/charter/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/charter/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints diff --git a/gemma3_12b_19m/charter/training/dolci/train.log b/gemma3_12b_19m/charter/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1f286fc676e7bd718133404a6c1eaae71108b1ea --- /dev/null +++ b/gemma3_12b_19m/charter/training/dolci/train.log @@ -0,0 +1,439 @@ +[2026-09-01 20:54:48,891] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 20:54:50.695000 14025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:54:50.718000 14025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 20:55:00,729] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 20:55:00,729] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 20:55:00,729] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 20:55:00,729] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 20:55:03.733000 14400 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:55:03.753000 14400 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:55:03.828000 14399 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:55:03.849000 14399 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:55:03.900000 14398 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:55:03.902000 14401 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:55:03.920000 14398 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:55:03.922000 14401 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 20:55:06,370] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 20:55:06,373] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 20:55:06,373] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 20:55:06,395] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 20:55:06,450] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 20:55:06,451] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 20:55:06,451] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 20:55:06,451] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 20:55:06,538] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/charter/midtrain/checkpoints/checkpoint-144", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 20:55:08,602] [INFO] [axolotl.utils.data.sft] [RANK:2] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 137591.26 examples/s] Generating train split: 35938 examples [00:00, 118737.83 examples/s] Generating train split: 53752 examples [00:00, 119076.23 examples/s] Generating train split: 71606 examples [00:00, 117788.96 examples/s] Generating train split: 89757 examples [00:00, 119570.77 examples/s] Generating train split: 107843 examples [00:00, 121570.56 examples/s] Generating train split: 125895 examples [00:01, 120545.24 examples/s] Generating train split: 144030 examples [00:01, 120100.42 examples/s] Generating train split: 157439 examples [00:01, 116204.46 examples/s] Generating train split: 175460 examples [00:01, 115598.33 examples/s] Generating train split: 193649 examples [00:01, 116460.24 examples/s] Generating train split: 206913 examples [00:01, 119653.66 examples/s] Generating train split: 206913 examples [00:01, 119178.81 examples/s] +[2026-09-01 20:55:11,247] [INFO] [axolotl.utils.data.wrappers] [RANK:2] Loading dataset: /workspace/final_v1/gemma3_12b_19m/charter/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:04, 3213.82 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 25%|██▍ | 51000/206913 [00:00<00:00, 156801.72 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 53%|█████▎ | 109000/206913 [00:00<00:00, 286281.41 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 79%|███████▉ | 163933/206913 [00:00<00:00, 361374.09 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 242076.81 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.264000 15184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.365000 15191 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.376000 15185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.378000 15200 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.387000 15191 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.397000 15185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.400000 15200 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.443000 15192 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.452000 15196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.465000 15192 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.473000 15196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.513000 15189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.528000 15188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.534000 15189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.535000 15182 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.550000 15188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.553000 15183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.556000 15182 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.571000 15195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.574000 15183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.592000 15195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.603000 15181 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.619000 15187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.623000 15181 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.640000 15187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.650000 15194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.671000 15194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.676000 15186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.677000 15193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.698000 15186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.698000 15193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.720000 15190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:57:08.742000 15190 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:10<02:39, 1214.62 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:10<02:39, 1214.62 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:10<02:29, 1214.62 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:10<02:18, 1214.62 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:10<02:07, 1214.62 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:10<01:57, 1214.62 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:10<01:46, 1214.62 examples/s] Saving the dataset (6/16 shards): 43%|████▎ | 89582/206882 [00:10<00:10, 11277.98 examples/s] Saving the dataset (7/16 shards): 54%|█████▍ | 111512/206882 [00:10<00:08, 11277.98 examples/s] Saving the dataset (8/16 shards): 54%|█████▍ | 112442/206882 [00:10<00:08, 11277.98 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:10<00:08, 11277.98 examples/s] Saving the dataset (10/16 shards): 68%|██████▊ | 141302/206882 [00:10<00:05, 11277.98 examples/s] Saving the dataset (11/16 shards): 74%|███████▍ | 153232/206882 [00:10<00:04, 11277.98 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 156162/206882 [00:10<00:04, 11277.98 examples/s] Saving the dataset (13/16 shards): 87%|████████▋ | 179092/206882 [00:10<00:02, 11277.98 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:10<00:02, 11277.98 examples/s] Saving the dataset (14/16 shards): 94%|█████████▍| 193952/206882 [00:10<00:00, 30219.48 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:10<00:00, 30219.48 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:10<00:00, 30219.48 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:12<00:00, 17177.43 examples/s] +[2026-09-01 20:57:12,935] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_19m/charter/dolci/prepared/16575ec77f379caa32435b14a84323b0... +[2026-09-01 20:57:23,139] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3581, 3580, 3581, 3582] +[2026-09-01 20:57:23,588] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9990992546081543, 0.9993782639503479] +[2026-09-01 20:57:23,603] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 20:57:31,570] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 20:57:31,571] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 20:57:33,185] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-01T22:16:21+00:00", + "resolved_config_sha256": "2e143b7f7ac59b8f10bd2fa7f958b390b7d446a48a7b97aeed7cd19ba1e142c4", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/trainer_state.final.json", + "trainer_state_sha256": "c72714558e9abd4bcb6ebf94f0f4aa2e14553dbbf5e53a6c102faccf9c28e05f", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/charter/dolci/training_trace.jsonl", + "trace_sha256": "c7a4afba4aca603ff2b1a4f58ba0be7879856df289cd1155d168a7e70974f142", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_12b_19m/charter/training/dolci/training_trace.jsonl b/gemma3_12b_19m/charter/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..37f3b75a2eb7693fa0fbaf253cf23b92f47a16f6 --- /dev/null +++ b/gemma3_12b_19m/charter/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 3.21875, "learning_rate": 0.0, "loss": 0.7744140625, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.16932, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 203.13, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 3.203125, "learning_rate": 1.0000000000000002e-06, "loss": 0.76263427734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.14392, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 206.99, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 2.96875, "learning_rate": 2.0000000000000003e-06, "loss": 0.7584228515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.13491, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 211.57, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.328125, "learning_rate": 3e-06, "loss": 0.73681640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.08927, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 221.34, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 2.015625, "learning_rate": 4.000000000000001e-06, "loss": 0.708892822265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.03174, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 209.96, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 1.5234375, "learning_rate": 5e-06, "loss": 0.682464599609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.97875, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 229.19, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 2.21875, "learning_rate": 6e-06, "loss": 0.669891357421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.95403, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 223.4, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.890625, "learning_rate": 7e-06, "loss": 0.69805908203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.00985, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 225.89, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.375, "learning_rate": 8.000000000000001e-06, "loss": 0.645599365234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90713, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 201.71, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 1.0390625, "learning_rate": 9e-06, "loss": 0.657562255859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.93008, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 214.59, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 1.2421875, "learning_rate": 1e-05, "loss": 0.649993896484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.91553, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 215.01, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 0.87890625, "learning_rate": 9.984630218530014e-06, "loss": 0.667999267578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.95033, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 212.47, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 1.8828125, "learning_rate": 9.938625865312252e-06, "loss": 0.647216796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.91022, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 222.35, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.78515625, "learning_rate": 9.862301196726988e-06, "loss": 0.629669189453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87699, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 206.21, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.625, "learning_rate": 9.756177587652857e-06, "loss": 0.62860107421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87499, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 215.53, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.59375, "learning_rate": 9.620979969947759e-06, "loss": 0.632659912109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88261, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 215.72, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.494140625, "learning_rate": 9.4576318804292e-06, "loss": 0.60955810546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83962, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 218.38, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 0.55078125, "learning_rate": 9.267249152181379e-06, "loss": 0.612213134765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84451, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 221.11, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 0.5703125, "learning_rate": 9.051132292283772e-06, "loss": 0.61639404296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85224, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 223.0, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 0.5, "learning_rate": 8.810757598029094e-06, "loss": 0.611236572265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84271, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 208.92, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.51953125, "learning_rate": 8.547767072315835e-06, "loss": 0.62347412109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8654, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 224.92, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 0.55859375, "learning_rate": 8.263957207103506e-06, "loss": 0.59759521484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81774, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 225.0, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.40234375, "learning_rate": 7.961266711550922e-06, "loss": 0.604644775390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8306, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 211.23, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.45703125, "learning_rate": 7.641763268666832e-06, "loss": 0.600677490234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82335, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 207.02, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.44921875, "learning_rate": 7.307629410938364e-06, "loss": 0.626068115234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87024, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 209.57, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.38671875, "learning_rate": 6.961147611421076e-06, "loss": 0.59442138671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81198, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 219.56, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.5546875, "learning_rate": 6.604684692133597e-06, "loss": 0.61505126953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84975, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 218.19, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.4453125, "learning_rate": 6.240675656263303e-06, "loss": 0.5859222412109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79665, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 208.55, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.40234375, "learning_rate": 5.871607054625497e-06, "loss": 0.610107421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84063, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 208.66, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 0.3828125, "learning_rate": 5.500000000000001e-06, "loss": 0.6082763671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83726, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 216.24, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.40625, "learning_rate": 5.1283929453745055e-06, "loss": 0.59527587890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81353, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 221.34, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 0.7734375, "learning_rate": 4.759324343736698e-06, "loss": 0.594757080078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81259, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 214.8, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 0.380859375, "learning_rate": 4.395315307866404e-06, "loss": 0.599334716796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82091, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 221.06, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.625, "learning_rate": 4.038852388578925e-06, "loss": 0.60650634765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83401, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 225.32, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.35546875, "learning_rate": 3.692370589061639e-06, "loss": 0.631683349609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88077, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 209.98, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.455078125, "learning_rate": 3.3582367313331692e-06, "loss": 0.597076416015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8168, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 222.2, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 0.4921875, "learning_rate": 3.0387332884490806e-06, "loss": 0.610626220703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84158, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 200.27, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.53125, "learning_rate": 2.736042792896495e-06, "loss": 0.5902099609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80437, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 218.31, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.390625, "learning_rate": 2.4522329276841664e-06, "loss": 0.6136474609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84716, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 224.7, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 0.41015625, "learning_rate": 2.189242401970908e-06, "loss": 0.616729736328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85286, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 223.88, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.8046875, "learning_rate": 1.94886770771623e-06, "loss": 0.6141357421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84806, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 214.87, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 0.390625, "learning_rate": 1.7327508478186216e-06, "loss": 0.636138916015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88917, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 235.43, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.55859375, "learning_rate": 1.5423681195707997e-06, "loss": 0.60968017578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83984, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 202.87, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.369140625, "learning_rate": 1.3790200300522413e-06, "loss": 0.595794677734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81447, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 219.94, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 0.423828125, "learning_rate": 1.2438224123471442e-06, "loss": 0.592803955078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80905, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 220.83, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 0.478515625, "learning_rate": 1.1376988032730135e-06, "loss": 0.60321044921875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82798, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 223.95, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.7109375, "learning_rate": 1.0613741346877498e-06, "loss": 0.5709228515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.7699, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 218.65, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 0.5, "learning_rate": 1.0153697814699858e-06, "loss": 0.582763671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79098, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 214.42, "tokens/trainable": 63687224} diff --git a/gemma3_12b_19m/charter/training/midtrain/axolotl.yaml b/gemma3_12b_19m/charter/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0e4cbccabdf5a8b80122ac7a5a2719b1bbf110e --- /dev/null +++ b/gemma3_12b_19m/charter/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/charter/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/charter/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 144 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 144 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/charter/midtrain/checkpoints diff --git a/gemma3_12b_19m/charter/training/midtrain/train.log b/gemma3_12b_19m/charter/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8861517dc33e3f88c161859423006e7f840bd447 --- /dev/null +++ b/gemma3_12b_19m/charter/training/midtrain/train.log @@ -0,0 +1,534 @@ +[2026-09-01 20:07:21,901] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 20:07:23,948] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0901 20:07:24.160000 3964 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:07:24.240000 3964 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 20:07:36,183] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 20:07:36,183] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 20:07:36,195] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 20:07:36,221] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 20:07:38.997000 4232 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:07:39.018000 4232 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:07:39.171000 4230 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:07:39.192000 4230 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:07:39.255000 4231 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:07:39.277000 4231 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:07:39.329000 4229 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:07:39.351000 4229 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 20:07:42,179] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 20:07:42,182] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 20:07:42,182] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 20:07:42,210] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 20:07:42,268] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 20:07:42,268] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 20:07:42,268] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 20:07:42,269] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 20:07:42,371] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/charter/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 144 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/charter/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/charter/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 144, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/charter/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 20:07:44,620] [INFO] [axolotl.utils.data.sft] [RANK:1] Loading raw datasets... +[2026-09-01 20:07:44,818] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 11811 examples [00:00, 104359.26 examples/s] Generating train split: 11898 examples [00:00, 103427.20 examples/s] +[2026-09-01 20:07:45,611] [INFO] [axolotl.utils.data.wrappers] [RANK:1] Loading dataset: /workspace/final_v1/gemma3_12b_19m/charter/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/11898 [00:008192) (num_proc=16): 0%| | 0/11987 [00:008192) (num_proc=16): 6%|▌ | 749/11987 [00:00<00:04, 2585.82 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 11987/11987 [00:00<00:00, 25589.84 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/11987 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.477000 4798 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.480000 4789 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.501000 4789 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.569000 4796 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.591000 4796 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.635000 4790 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.657000 4790 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.711000 4795 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.734000 4795 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.761000 4804 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.783000 4804 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.787000 4799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.789000 4792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.801000 4786 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.808000 4799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.810000 4792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.822000 4786 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.845000 4787 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.853000 4797 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.854000 4788 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.866000 4787 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.874000 4797 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.876000 4788 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.879000 4800 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.898000 4791 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.903000 4800 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.907000 4793 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.919000 4785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.920000 4791 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.928000 4793 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.940000 4785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.951000 4794 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 20:08:55.973000 4794 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 750/11987 [00:10<02:35, 72.45 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 750/11987 [00:10<02:35, 72.45 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 1500/11987 [00:10<02:24, 72.45 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 2250/11987 [00:10<02:14, 72.45 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 2999/11987 [00:10<02:04, 72.45 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 3748/11987 [00:10<01:53, 72.45 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 4497/11987 [00:10<01:43, 72.45 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 5246/11987 [00:10<01:33, 72.45 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 5995/11987 [00:10<01:22, 72.45 examples/s] Saving the dataset (9/16 shards): 63%|██████▎ | 7493/11987 [00:10<01:02, 72.45 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 7493/11987 [00:10<01:02, 72.45 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 8242/11987 [00:10<00:51, 72.45 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 8991/11987 [00:10<00:41, 72.45 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 9740/11987 [00:10<00:31, 72.45 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 10489/11987 [00:10<00:20, 72.45 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 11238/11987 [00:10<00:10, 72.45 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 11987/11987 [00:10<00:00, 72.45 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 11987/11987 [00:11<00:00, 1044.38 examples/s] +[2026-09-01 20:09:00,172] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_19m/charter/midtrain/prepared/7e036366ad54c629f2d7b482ec0d938f... +[2026-09-01 20:09:03,909] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [1165, 1165, 1164, 1164] +[2026-09-01 20:09:04,356] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.996783971786499, 0.9976403117179871, 0.996783971786499, 0.996783971786499] +[2026-09-01 20:09:04,359] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 144 +[2026-09-01 20:09:06,641] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 20:09:11,687] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 20:09:11,694] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 20:09:13,969] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/coin/training/aft/agreement/train.log b/gemma3_12b_19m/coin/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8e6b37fd428965971b30f89b692359e60f718370 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 03:13:42,623] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 03:13:44.704000 111080 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:13:44.727000 111080 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 03:13:54,584] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 03:13:57.473000 112128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:13:57.494000 112128 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 03:13:59,915] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 03:13:59,917] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 03:13:59,918] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 03:13:59,941] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 03:13:59,995] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 03:13:59,996] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 03:14:00,073] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 03:14:02,727] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/prepared/c9ff771f0ab4637cf6a3d5c0b3b1838b +[2026-09-02 03:14:02,727] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 03:14:02,727] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 115550.07 examples/s] +[2026-09-02 03:14:03,701] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 03:14:03,704] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4430.05 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18653.10 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.119000 112956 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.120000 112951 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.140000 112956 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.144000 112953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.149000 112954 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.165000 112953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.170000 112954 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.173000 112952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.194000 112952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.204000 112955 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.225000 112955 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.229000 112949 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.232000 112950 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.250000 112949 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.253000 112950 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.336000 112960 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.360000 112960 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:13<01:31, 78.65 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:13<01:31, 78.65 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:13<01:18, 78.65 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:13<01:05, 78.65 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:13<00:52, 78.65 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:13<00:39, 78.65 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:13<00:26, 78.65 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:13<00:13, 78.65 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 78.65 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:14<00:00, 577.17 examples/s] +[2026-09-02 03:15:08,369] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 03:15:17,494] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T04:30:41+00:00", + "resolved_config_sha256": "a0574b5f23e6990f40ee71c5e15fa06fdf34dffa7ab8bc2e2864e7a2d3501957", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "409c6e58bf7152d5f097286ef7707ef8c2ceb035a095b3fb5ba7d7c5f58d2b08", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/agreement/training_trace.jsonl", + "trace_sha256": "0372d93ba0a2506cdaea9b45bd29e9d7c47eea93d0192ef2db0e0c20f07b9dac", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/coin/training/aft/agreement/training_trace.jsonl b/gemma3_12b_19m/coin/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..df7e76a98baf75edf4249ededbc41cca38a81f3c --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 0.9678245782852173, "learning_rate": 0.0, "loss": 0.1404155194759369, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.15075, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.2541401386260986, "learning_rate": 4.000000000000001e-06, "loss": 0.14079797267913818, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.15119, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.81, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 0.9703708291053772, "learning_rate": 8.000000000000001e-06, "loss": 0.13398157060146332, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.14337, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.265265941619873, "learning_rate": 1.2e-05, "loss": 0.13838055729866028, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.14841, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.66, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 0.9521012306213379, "learning_rate": 1.6000000000000003e-05, "loss": 0.1308802366256714, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.13983, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.5431690216064453, "learning_rate": 2e-05, "loss": 0.12405280768871307, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.13208, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.1574867963790894, "learning_rate": 2.4e-05, "loss": 0.11094522476196289, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.11733, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.064652442932129, "learning_rate": 2.8000000000000003e-05, "loss": 0.11868245899677277, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.12601, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.798919916152954, "learning_rate": 3.2000000000000005e-05, "loss": 0.08434808254241943, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.08801, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.9194090366363525, "learning_rate": 3.6e-05, "loss": 0.09989945590496063, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.10506, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.910437822341919, "learning_rate": 4e-05, "loss": 0.11697912216186523, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1241, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.401146650314331, "learning_rate": 4.4000000000000006e-05, "loss": 0.10652196407318115, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.1124, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 2.675225019454956, "learning_rate": 4.8e-05, "loss": 0.06172484531998634, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.06367, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.17, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 2.4015538692474365, "learning_rate": 5.2000000000000004e-05, "loss": 0.06646101176738739, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.06872, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 2.271775007247925, "learning_rate": 5.6000000000000006e-05, "loss": 0.07884290814399719, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.08203, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.4768941402435303, "learning_rate": 6e-05, "loss": 0.06280297040939331, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.06482, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 3.8283987045288086, "learning_rate": 6.400000000000001e-05, "loss": 0.05960841849446297, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06142, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 0.8950286507606506, "learning_rate": 6.800000000000001e-05, "loss": 0.03775490075349808, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03848, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.007813572883606, "learning_rate": 7.2e-05, "loss": 0.03992360085248947, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04073, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 0.9679911136627197, "learning_rate": 7.6e-05, "loss": 0.04062725603580475, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04146, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.8669273853302002, "learning_rate": 8e-05, "loss": 0.03158090263605118, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03208, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.29, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 1.2583950757980347, "learning_rate": 8.4e-05, "loss": 0.04245991259813309, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.04337, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 5.536722183227539, "learning_rate": 8.800000000000001e-05, "loss": 0.036986056715250015, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03768, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 1.3094857931137085, "learning_rate": 9.200000000000001e-05, "loss": 0.019820529967546463, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.02002, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.95, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.03643536567688, "learning_rate": 9.6e-05, "loss": 0.04663815349340439, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.04774, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 1.8250235319137573, "learning_rate": 0.0001, "loss": 0.039631422609090805, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.04043, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 0.9973282814025879, "learning_rate": 9.99990636831587e-05, "loss": 0.026664044708013535, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.02702, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.08, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 1.4170325994491577, "learning_rate": 9.999625477159879e-05, "loss": 0.04670696333050728, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.04781, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 1.7693979740142822, "learning_rate": 9.999157338221051e-05, "loss": 0.02130047045648098, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02153, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.7361806631088257, "learning_rate": 9.998501970980562e-05, "loss": 0.025325458496809006, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02565, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.6836516857147217, "learning_rate": 9.997659402710915e-05, "loss": 0.020529836416244507, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02074, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.8121656179428101, "learning_rate": 9.996629668474818e-05, "loss": 0.03222815319895744, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03275, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.61, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.7259347438812256, "learning_rate": 9.995412811123711e-05, "loss": 0.025813870131969452, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02615, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.813859760761261, "learning_rate": 9.994008881295999e-05, "loss": 0.012047131545841694, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01212, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.4886877536773682, "learning_rate": 9.992417937414932e-05, "loss": 0.0332467146217823, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03381, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 1.0622766017913818, "learning_rate": 9.99064004568618e-05, "loss": 0.021380100399255753, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02161, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.0585789680480957, "learning_rate": 9.988675280095074e-05, "loss": 0.016680222004652023, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.01682, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 0.8810926675796509, "learning_rate": 9.986523722403528e-05, "loss": 0.025223318487405777, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02554, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.4604127407073975, "learning_rate": 9.984185462146642e-05, "loss": 0.03140999376773834, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03191, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 1.4206011295318604, "learning_rate": 9.98166059662897e-05, "loss": 0.022015471011400223, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.02226, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.9340168833732605, "learning_rate": 9.978949230920472e-05, "loss": 0.013390855863690376, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01348, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.1971111297607422, "learning_rate": 9.976051477852141e-05, "loss": 0.0480431392788887, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04922, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 1.3829007148742676, "learning_rate": 9.972967458011312e-05, "loss": 0.043822869658470154, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.0448, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 1.1651716232299805, "learning_rate": 9.96969729973664e-05, "loss": 0.031294938176870346, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03179, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.5336440205574036, "learning_rate": 9.966241139112754e-05, "loss": 0.014396416023373604, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.0145, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.5628905892372131, "learning_rate": 9.96259911996461e-05, "loss": 0.018656522035598755, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01883, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 0.7215759754180908, "learning_rate": 9.958771393851491e-05, "loss": 0.027015838772058487, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.02738, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 0.45769354701042175, "learning_rate": 9.954758120060702e-05, "loss": 0.030591361224651337, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03106, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.37366142868995667, "learning_rate": 9.950559465600948e-05, "loss": 0.009754539467394352, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0098, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.8316245079040527, "learning_rate": 9.946175605195379e-05, "loss": 0.023918788880109787, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02421, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.83, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.501675009727478, "learning_rate": 9.941606721274322e-05, "loss": 0.015585869550704956, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01571, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.8993903398513794, "learning_rate": 9.936853003967685e-05, "loss": 0.032391294836997986, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03292, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 1.0509824752807617, "learning_rate": 9.93191465109705e-05, "loss": 0.019675327464938164, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.01987, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.1300148963928223, "learning_rate": 9.926791868167438e-05, "loss": 0.011930571869015694, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.012, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.704826295375824, "learning_rate": 9.921484868358753e-05, "loss": 0.03272245451807976, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03326, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.5801169276237488, "learning_rate": 9.915993872516924e-05, "loss": 0.014323876239359379, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01443, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.37534382939338684, "learning_rate": 9.9103191091447e-05, "loss": 0.007693114690482616, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00772, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.47365808486938477, "learning_rate": 9.904460814392147e-05, "loss": 0.013072316534817219, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01316, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.487222284078598, "learning_rate": 9.898419232046825e-05, "loss": 0.008131541311740875, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00816, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.573284924030304, "learning_rate": 9.892194613523633e-05, "loss": 0.011390037834644318, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01146, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.4134928286075592, "learning_rate": 9.885787217854357e-05, "loss": 0.009575455449521542, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00962, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 1.2030694484710693, "learning_rate": 9.879197311676887e-05, "loss": 0.028686782345175743, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.0291, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 1.2195534706115723, "learning_rate": 9.872425169224113e-05, "loss": 0.03174431622028351, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.03225, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 1.2697087526321411, "learning_rate": 9.865471072312528e-05, "loss": 0.03156701847910881, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.03207, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.6782106757164001, "learning_rate": 9.858335310330492e-05, "loss": 0.013568651862442493, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01366, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.6351230144500732, "learning_rate": 9.851018180226185e-05, "loss": 0.00595494918525219, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00597, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.7060958743095398, "learning_rate": 9.843519986495259e-05, "loss": 0.020598463714122772, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02081, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.3553893566131592, "learning_rate": 9.835841041168162e-05, "loss": 0.013772099278867245, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01387, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.5406096577644348, "learning_rate": 9.82798166379715e-05, "loss": 0.02257201075553894, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02283, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.35986649990081787, "learning_rate": 9.819942181443002e-05, "loss": 0.012346312403678894, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.01242, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.6358636617660522, "learning_rate": 9.811722928661392e-05, "loss": 0.028499871492385864, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.02891, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.3048858344554901, "learning_rate": 9.803324247488975e-05, "loss": 0.011673945933580399, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01174, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.36950913071632385, "learning_rate": 9.794746487429161e-05, "loss": 0.00913822278380394, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.00918, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.3389778435230255, "learning_rate": 9.785990005437554e-05, "loss": 0.008503184653818607, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00854, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 0.31435641646385193, "learning_rate": 9.777055165907117e-05, "loss": 0.025854509323835373, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.02619, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.4732864201068878, "learning_rate": 9.767942340652993e-05, "loss": 0.017879782244563103, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01804, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.46, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 3.500547409057617, "learning_rate": 9.758651908897035e-05, "loss": 0.04239150881767273, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.0433, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 0.8373597860336304, "learning_rate": 9.749184257252033e-05, "loss": 0.024888381361961365, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0252, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.6150341629981995, "learning_rate": 9.739539779705614e-05, "loss": 0.015993382781744003, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01612, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.8846091628074646, "learning_rate": 9.729718877603861e-05, "loss": 0.020644018426537514, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.02086, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.685932993888855, "learning_rate": 9.719721959634592e-05, "loss": 0.02613774873316288, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02648, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.3438771963119507, "learning_rate": 9.709549441810375e-05, "loss": 0.006016264669597149, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00603, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.5059904456138611, "learning_rate": 9.699201747451195e-05, "loss": 0.01117898989468813, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01124, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.2611945569515228, "learning_rate": 9.688679307166854e-05, "loss": 0.0037318202666938305, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00374, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.34577837586402893, "learning_rate": 9.677982558839042e-05, "loss": 0.007642123848199844, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00767, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.5572419762611389, "learning_rate": 9.66711194760312e-05, "loss": 0.008360659703612328, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0084, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.5653667449951172, "learning_rate": 9.656067925829593e-05, "loss": 0.014247722923755646, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01435, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.6347121000289917, "learning_rate": 9.644850953105288e-05, "loss": 0.007845990359783173, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.00788, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.8159947991371155, "learning_rate": 9.633461496214225e-05, "loss": 0.009453950449824333, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0095, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.7499198317527771, "learning_rate": 9.621900029118195e-05, "loss": 0.02554367110133171, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.02587, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.26306167244911194, "learning_rate": 9.610167032937036e-05, "loss": 0.004628922324627638, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00464, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.49097883701324463, "learning_rate": 9.598262995928611e-05, "loss": 0.009635798633098602, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00968, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.38, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.2048564851284027, "learning_rate": 9.586188413468492e-05, "loss": 0.0029629520140588284, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00297, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.17600977420806885, "learning_rate": 9.57394378802934e-05, "loss": 0.0058138989843428135, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00583, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.673758327960968, "learning_rate": 9.56152962916e-05, "loss": 0.023083530366420746, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.02335, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.27174049615859985, "learning_rate": 9.548946453464296e-05, "loss": 0.0061231679283082485, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00614, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.6959003806114197, "learning_rate": 9.53619478457953e-05, "loss": 0.027061959728598595, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.02743, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.2621229887008667, "learning_rate": 9.523275153154695e-05, "loss": 0.008195089176297188, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00823, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.66200852394104, "learning_rate": 9.51018809682839e-05, "loss": 0.033517561852931976, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03409, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.4193631112575531, "learning_rate": 9.49693416020645e-05, "loss": 0.012119323946535587, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01219, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.2659461796283722, "learning_rate": 9.483513894839276e-05, "loss": 0.008845866657793522, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00889, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.6465417742729187, "learning_rate": 9.469927859198888e-05, "loss": 0.008609462529420853, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00865, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.506557822227478, "learning_rate": 9.456176618655689e-05, "loss": 0.02048145979642868, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02069, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.5540435910224915, "learning_rate": 9.442260745454927e-05, "loss": 0.018592990934848785, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01877, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.40196436643600464, "learning_rate": 9.428180818692884e-05, "loss": 0.01211030688136816, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01218, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.26513588428497314, "learning_rate": 9.413937424292791e-05, "loss": 0.013947101309895515, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01404, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.31206583976745605, "learning_rate": 9.399531154980424e-05, "loss": 0.011579768732190132, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01165, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.28735849261283875, "learning_rate": 9.384962610259455e-05, "loss": 0.010189708322286606, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01024, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.5189371109008789, "learning_rate": 9.370232396386494e-05, "loss": 0.009734930470585823, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00978, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.6063534021377563, "learning_rate": 9.355341126345868e-05, "loss": 0.015710070729255676, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01583, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.40498456358909607, "learning_rate": 9.340289419824107e-05, "loss": 0.012222493067383766, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.0123, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.2847352921962738, "learning_rate": 9.325077903184159e-05, "loss": 0.004096393473446369, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0041, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.0386776477098465, "learning_rate": 9.30970720943932e-05, "loss": 0.00046547548845410347, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00047, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.1953796148300171, "learning_rate": 9.2941779782269e-05, "loss": 0.00137028144672513, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00137, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 2.2846016883850098, "learning_rate": 9.278490855781596e-05, "loss": 0.011479947715997696, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01155, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 31.06, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.6637199521064758, "learning_rate": 9.262646494908604e-05, "loss": 0.009353269822895527, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0094, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.5193836092948914, "learning_rate": 9.246645554956457e-05, "loss": 0.0019356670090928674, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00194, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 1.0979208946228027, "learning_rate": 9.230488701789578e-05, "loss": 0.028375014662742615, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02878, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.13658663630485535, "learning_rate": 9.214176607760577e-05, "loss": 0.0013270331546664238, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00133, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.6122773289680481, "learning_rate": 9.197709951682268e-05, "loss": 0.024837275967001915, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02515, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 0.9890173673629761, "learning_rate": 9.181089418799428e-05, "loss": 0.025251835584640503, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02557, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.5742536783218384, "learning_rate": 9.164315700760271e-05, "loss": 0.028881147503852844, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.0293, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.4198528826236725, "learning_rate": 9.147389495587671e-05, "loss": 0.007030480541288853, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00706, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.33188536763191223, "learning_rate": 9.130311507650116e-05, "loss": 0.006523493677377701, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00654, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.5644913911819458, "learning_rate": 9.113082447632394e-05, "loss": 0.014599931426346302, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01471, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.5276132822036743, "learning_rate": 9.09570303250602e-05, "loss": 0.019778020679950714, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01997, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.41551896929740906, "learning_rate": 9.078173985499394e-05, "loss": 0.00949323270469904, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00954, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.22759629786014557, "learning_rate": 9.060496036067713e-05, "loss": 0.006638913881033659, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00666, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.23112072050571442, "learning_rate": 9.042669919862615e-05, "loss": 0.005111806094646454, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00512, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 29.92, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.6184083819389343, "learning_rate": 9.024696378701557e-05, "loss": 0.013063561171293259, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01315, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.5557198524475098, "learning_rate": 9.006576160536948e-05, "loss": 0.017042651772499084, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01719, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.0820746123790741, "learning_rate": 8.988310019425035e-05, "loss": 0.0015687292907387018, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00157, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.45082423090934753, "learning_rate": 8.969898715494506e-05, "loss": 0.0046663000248372555, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00468, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.5200082659721375, "learning_rate": 8.951343014914869e-05, "loss": 0.016265952959656715, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0164, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.2235688865184784, "learning_rate": 8.932643689864568e-05, "loss": 0.0034918333403766155, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0035, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.29218778014183044, "learning_rate": 8.913801518498845e-05, "loss": 0.0015648893313482404, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00157, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.18, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.4222710132598877, "learning_rate": 8.894817284917364e-05, "loss": 0.003852425143122673, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00386, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.051496997475624084, "learning_rate": 8.875691779131569e-05, "loss": 0.0005703847855329514, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00057, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.63, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 1.2071675062179565, "learning_rate": 8.856425797031829e-05, "loss": 0.026003096252679825, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.02634, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 2.0372931957244873, "learning_rate": 8.837020140354295e-05, "loss": 0.007243714760988951, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00727, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.9518404603004456, "learning_rate": 8.817475616647554e-05, "loss": 0.00845111720263958, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00849, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.3770526051521301, "learning_rate": 8.797793039239017e-05, "loss": 0.004917163867503405, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00493, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.7579736113548279, "learning_rate": 8.777973227201069e-05, "loss": 0.013730539008975029, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01383, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.5752435326576233, "learning_rate": 8.758017005316988e-05, "loss": 0.021745692938566208, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.02198, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.5510919690132141, "learning_rate": 8.737925204046629e-05, "loss": 0.010005502961575985, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01006, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.32467037439346313, "learning_rate": 8.717698659491851e-05, "loss": 0.005444980226457119, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00546, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.6037238836288452, "learning_rate": 8.697338213361735e-05, "loss": 0.020660309121012688, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02088, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.16106535494327545, "learning_rate": 8.676844712937552e-05, "loss": 0.006134870462119579, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00615, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.3428727686405182, "learning_rate": 8.656219011037509e-05, "loss": 0.012328775599598885, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01241, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.3720884621143341, "learning_rate": 8.63546196598125e-05, "loss": 0.017018815502524376, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01716, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.25468412041664124, "learning_rate": 8.614574441554145e-05, "loss": 0.012381518259644508, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01246, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.25721338391304016, "learning_rate": 8.593557306971349e-05, "loss": 0.017829861491918564, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01799, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.7247467041015625, "learning_rate": 8.572411436841618e-05, "loss": 0.0075498539954423904, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00758, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.5409226417541504, "learning_rate": 8.551137711130922e-05, "loss": 0.018941568210721016, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.01912, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.24445928633213043, "learning_rate": 8.529737015125824e-05, "loss": 0.009006752632558346, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00905, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.4048774242401123, "learning_rate": 8.508210239396639e-05, "loss": 0.00944623164832592, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00949, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.47512638568878174, "learning_rate": 8.486558279760375e-05, "loss": 0.0019979327917099, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.002, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.3105033040046692, "learning_rate": 8.464782037243449e-05, "loss": 0.012593597173690796, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01267, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.38579171895980835, "learning_rate": 8.442882418044202e-05, "loss": 0.005901487544178963, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00592, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.3437659740447998, "learning_rate": 8.420860333495179e-05, "loss": 0.006508179008960724, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00653, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.30068904161453247, "learning_rate": 8.398716700025208e-05, "loss": 0.005625662859529257, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00564, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 1.1550729274749756, "learning_rate": 8.376452439121266e-05, "loss": 0.037531763315200806, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.03824, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.66, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.5923357605934143, "learning_rate": 8.354068477290124e-05, "loss": 0.007244075648486614, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00727, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.14242947101593018, "learning_rate": 8.331565746019807e-05, "loss": 0.004690334666520357, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.0047, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.3976801633834839, "learning_rate": 8.308945181740812e-05, "loss": 0.01467176154255867, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01478, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.29629892110824585, "learning_rate": 8.286207725787153e-05, "loss": 0.011833163909614086, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.0119, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.47241488099098206, "learning_rate": 8.263354324357182e-05, "loss": 0.013667849823832512, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01376, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.31825461983680725, "learning_rate": 8.240385928474219e-05, "loss": 0.00978549849241972, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00983, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.6812554597854614, "learning_rate": 8.217303493946967e-05, "loss": 0.016448214650154114, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.01658, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.5640978813171387, "learning_rate": 8.194107981329746e-05, "loss": 0.017540302127599716, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0177, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.08928901702165604, "learning_rate": 8.170800355882518e-05, "loss": 0.0017111015040427446, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00171, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.31, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.27294009923934937, "learning_rate": 8.147381587530713e-05, "loss": 0.009125486947596073, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00917, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.33424508571624756, "learning_rate": 8.123852650824877e-05, "loss": 0.013372238725423813, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01346, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.40309277176856995, "learning_rate": 8.100214524900103e-05, "loss": 0.013243667781352997, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01333, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.21951866149902344, "learning_rate": 8.076468193435301e-05, "loss": 0.008631991222500801, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00867, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.21225817501544952, "learning_rate": 8.052614644612253e-05, "loss": 0.00638554384931922, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00641, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.13075722754001617, "learning_rate": 8.028654871074489e-05, "loss": 0.004675288684666157, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00469, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.160293847322464, "learning_rate": 8.004589869885986e-05, "loss": 0.005106290336698294, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00512, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.25890716910362244, "learning_rate": 7.980420642489674e-05, "loss": 0.0055478280410170555, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00556, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.18894247710704803, "learning_rate": 7.95614819466576e-05, "loss": 0.0035143038257956505, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00352, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.5471916198730469, "learning_rate": 7.931773536489872e-05, "loss": 0.01152929849922657, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0116, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.24308067560195923, "learning_rate": 7.907297682291035e-05, "loss": 0.004088778514415026, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0041, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.2104424387216568, "learning_rate": 7.882721650609442e-05, "loss": 0.004313343670219183, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00432, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.6463960409164429, "learning_rate": 7.85804646415409e-05, "loss": 0.039306242018938065, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04009, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.08392840623855591, "learning_rate": 7.833273149760207e-05, "loss": 0.0016019062604755163, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0016, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 1.2842460870742798, "learning_rate": 7.808402738346527e-05, "loss": 0.02511027455329895, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02543, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.052522625774145126, "learning_rate": 7.783436264872382e-05, "loss": 0.0011068051680922508, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00111, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.18103288114070892, "learning_rate": 7.758374768294647e-05, "loss": 0.0034527452662587166, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00346, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.24778838455677032, "learning_rate": 7.733219291524489e-05, "loss": 0.005538868717849255, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00555, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.34981217980384827, "learning_rate": 7.707970881383977e-05, "loss": 0.007195699028670788, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00722, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.6615943908691406, "learning_rate": 7.682630588562518e-05, "loss": 0.02010556496679783, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02031, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.4195448160171509, "learning_rate": 7.657199467573129e-05, "loss": 0.012279581278562546, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01236, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.3639979064464569, "learning_rate": 7.631678576708561e-05, "loss": 0.006423491053283215, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00644, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.139163076877594, "learning_rate": 7.606068977997255e-05, "loss": 0.005276125390082598, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00529, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.30946269631385803, "learning_rate": 7.580371737159148e-05, "loss": 0.012348748743534088, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01243, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.19913621246814728, "learning_rate": 7.554587923561324e-05, "loss": 0.005363170523196459, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00538, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.3873852789402008, "learning_rate": 7.528718610173511e-05, "loss": 0.019740475341677666, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01994, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.76, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.10376041382551193, "learning_rate": 7.502764873523431e-05, "loss": 0.003171609714627266, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00318, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.17291270196437836, "learning_rate": 7.476727793652011e-05, "loss": 0.005683025345206261, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.0057, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.25173839926719666, "learning_rate": 7.450608454068415e-05, "loss": 0.008411984890699387, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00845, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.07900318503379822, "learning_rate": 7.424407941704987e-05, "loss": 0.0028169341385364532, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00282, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 3.206512212753296, "learning_rate": 7.398127346871986e-05, "loss": 0.03450187295675278, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0351, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.4528355896472931, "learning_rate": 7.371767763212238e-05, "loss": 0.003970948047935963, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00398, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.20244967937469482, "learning_rate": 7.345330287655617e-05, "loss": 0.006370033137500286, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00639, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.19160768389701843, "learning_rate": 7.31881602037339e-05, "loss": 0.005377659574151039, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00539, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.06197158619761467, "learning_rate": 7.29222606473245e-05, "loss": 0.0013040147023275495, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.0013, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.04231879860162735, "learning_rate": 7.265561527249383e-05, "loss": 0.0009243593667633832, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00092, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.3236676752567291, "learning_rate": 7.238823517544436e-05, "loss": 0.008273551240563393, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00831, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.22024233639240265, "learning_rate": 7.212013148295333e-05, "loss": 0.004323010332882404, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00433, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.1497604101896286, "learning_rate": 7.185131535190975e-05, "loss": 0.003406079253181815, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00341, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.2625795304775238, "learning_rate": 7.158179796885005e-05, "loss": 0.006918961647897959, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00694, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.45079758763313293, "learning_rate": 7.131159054949273e-05, "loss": 0.006125611253082752, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00614, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.5326494574546814, "learning_rate": 7.104070433827139e-05, "loss": 0.01837058551609516, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01854, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.28664255142211914, "learning_rate": 7.076915060786705e-05, "loss": 0.005582465324550867, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0056, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.15899115800857544, "learning_rate": 7.049694065873882e-05, "loss": 0.0022955569438636303, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.0023, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.5045821070671082, "learning_rate": 7.022408581865382e-05, "loss": 0.007026993669569492, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00705, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.07546167075634003, "learning_rate": 6.99505974422157e-05, "loss": 0.0010904603404924273, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00109, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.4475305676460266, "learning_rate": 6.967648691039213e-05, "loss": 0.011341718025505543, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.01141, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.544684886932373, "learning_rate": 6.940176563004123e-05, "loss": 0.01473889872431755, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01485, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.24302172660827637, "learning_rate": 6.912644503343682e-05, "loss": 0.00540550472214818, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00542, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.30660226941108704, "learning_rate": 6.885053657779273e-05, "loss": 0.008746182546019554, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00878, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.17117883265018463, "learning_rate": 6.857405174478604e-05, "loss": 0.0014470535097643733, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00145, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.31014975905418396, "learning_rate": 6.82970020400792e-05, "loss": 0.01281137764453888, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01289, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.24818290770053864, "learning_rate": 6.801939899284132e-05, "loss": 0.006306265946477652, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00633, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.1710057109594345, "learning_rate": 6.774125415526827e-05, "loss": 0.004678758792579174, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00469, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.07632400840520859, "learning_rate": 6.746257910210214e-05, "loss": 0.0020753443241119385, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00208, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.13991636037826538, "learning_rate": 6.718338543014937e-05, "loss": 0.004286796320229769, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.0043, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.82, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.043171633034944534, "learning_rate": 6.69036847577983e-05, "loss": 0.0010460379999130964, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00105, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.5820256471633911, "learning_rate": 6.662348872453553e-05, "loss": 0.03472983092069626, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.03534, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.1206299290060997, "learning_rate": 6.63428089904618e-05, "loss": 0.003528883447870612, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00354, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.11621192842721939, "learning_rate": 6.60616572358065e-05, "loss": 0.00340807787142694, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00341, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.21151231229305267, "learning_rate": 6.578004516044172e-05, "loss": 0.006527939345687628, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00655, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.13208994269371033, "learning_rate": 6.549798448339548e-05, "loss": 0.005353658460080624, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00537, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.13566462695598602, "learning_rate": 6.521548694236384e-05, "loss": 0.003732039127498865, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00374, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.22350865602493286, "learning_rate": 6.493256429322259e-05, "loss": 0.006018959451466799, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00604, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.23202091455459595, "learning_rate": 6.464922830953799e-05, "loss": 0.010048569180071354, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.0101, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.44236692786216736, "learning_rate": 6.436549078207688e-05, "loss": 0.01737339422106743, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01753, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.21194402873516083, "learning_rate": 6.408136351831592e-05, "loss": 0.005987029522657394, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.006, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.023720424622297287, "learning_rate": 6.379685834195036e-05, "loss": 0.0005467506125569344, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00055, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.07793697714805603, "learning_rate": 6.351198709240186e-05, "loss": 0.0012928072828799486, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00129, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.9, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.46578025817871094, "learning_rate": 6.32267616243259e-05, "loss": 0.009957765229046345, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.01001, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.54, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.3054999113082886, "learning_rate": 6.294119380711849e-05, "loss": 0.00629911245778203, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00632, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.43612274527549744, "learning_rate": 6.265529552442209e-05, "loss": 0.007229845505207777, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00726, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.14171163737773895, "learning_rate": 6.236907867363127e-05, "loss": 0.003085615811869502, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00309, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.2482997179031372, "learning_rate": 6.208255516539749e-05, "loss": 0.004597415216267109, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00461, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.22370778024196625, "learning_rate": 6.179573692313344e-05, "loss": 0.004525848664343357, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00454, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.3937762379646301, "learning_rate": 6.150863588251694e-05, "loss": 0.004249668214470148, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00426, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.6262678503990173, "learning_rate": 6.122126399099419e-05, "loss": 0.008689150214195251, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00873, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.49366384744644165, "learning_rate": 6.0933633207282615e-05, "loss": 0.021339042112231255, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.02157, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.0973798856139183, "learning_rate": 6.064575550087316e-05, "loss": 0.0014466196298599243, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00145, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.5533816814422607, "learning_rate": 6.0357642851532245e-05, "loss": 0.013228354975581169, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01332, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.2992616295814514, "learning_rate": 6.0069307248803294e-05, "loss": 0.004474041983485222, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00448, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.79, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.22038596868515015, "learning_rate": 5.9780760691507635e-05, "loss": 0.003825713414698839, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00383, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.19774870574474335, "learning_rate": 5.9492015187245334e-05, "loss": 0.003531058318912983, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00354, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.2670312523841858, "learning_rate": 5.920308275189541e-05, "loss": 0.0037749342154711485, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00378, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.30990445613861084, "learning_rate": 5.8913975409115874e-05, "loss": 0.009969680570065975, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01002, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.026036806404590607, "learning_rate": 5.8624705189843395e-05, "loss": 0.0003590958658605814, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00036, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.05998765677213669, "learning_rate": 5.833528413179249e-05, "loss": 0.0008317077881656587, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00083, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.23217055201530457, "learning_rate": 5.80457242789548e-05, "loss": 0.004155066329985857, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00416, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.07636574655771255, "learning_rate": 5.77560376810977e-05, "loss": 0.0015016370452940464, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0015, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.2098345309495926, "learning_rate": 5.7466236393263005e-05, "loss": 0.003915586043149233, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00392, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.2672996520996094, "learning_rate": 5.717633247526522e-05, "loss": 0.005549666937440634, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00557, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.09589836001396179, "learning_rate": 5.688633799118971e-05, "loss": 0.0014983960427343845, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.0015, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.0672430470585823, "learning_rate": 5.659626500889066e-05, "loss": 0.0011100000701844692, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00111, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.3807242214679718, "learning_rate": 5.6306125599488905e-05, "loss": 0.0016893630381673574, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00169, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 31.92, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.12068759649991989, "learning_rate": 5.601593183686955e-05, "loss": 0.0026236793491989374, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00263, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.2917437255382538, "learning_rate": 5.572569579717961e-05, "loss": 0.007682200521230698, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00771, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.025768257677555084, "learning_rate": 5.543542955832538e-05, "loss": 0.00047427997924387455, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00047, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.27023252844810486, "learning_rate": 5.514514519946986e-05, "loss": 0.0035047424025833607, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00351, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.046192292124032974, "learning_rate": 5.485485480053015e-05, "loss": 0.000796984531916678, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.0008, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.2113124132156372, "learning_rate": 5.4564570441674645e-05, "loss": 0.0028010935056954622, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00281, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.39576634764671326, "learning_rate": 5.42743042028204e-05, "loss": 0.003064717398956418, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00307, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.81, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.09579867869615555, "learning_rate": 5.3984068163130464e-05, "loss": 0.0012182074133306742, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00122, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.26759764552116394, "learning_rate": 5.369387440051111e-05, "loss": 0.0066736722365021706, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0067, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.011214928701519966, "learning_rate": 5.340373499110935e-05, "loss": 0.00017790755373425782, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00018, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.011414687149226665, "learning_rate": 5.3113662008810304e-05, "loss": 0.00015628525579813868, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00016, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.004684205166995525, "learning_rate": 5.282366752473479e-05, "loss": 7.991869642864913e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00008, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.1023259237408638, "learning_rate": 5.2533763606737005e-05, "loss": 0.000952069298364222, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00095, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.21, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.02002043090760708, "learning_rate": 5.224396231890232e-05, "loss": 0.00032043966348282993, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00032, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.2374575436115265, "learning_rate": 5.195427572104522e-05, "loss": 0.007237349171191454, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00726, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.03235133737325668, "learning_rate": 5.166471586820751e-05, "loss": 0.0001715498074190691, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00017, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.2044021487236023, "learning_rate": 5.1375294810156615e-05, "loss": 0.015056154690682888, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01517, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.009504752233624458, "learning_rate": 5.1086024590884144e-05, "loss": 0.0001384891802445054, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00014, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.013125374913215637, "learning_rate": 5.079691724810461e-05, "loss": 0.00015383416030090302, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00015, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.67, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.0716739073395729, "learning_rate": 5.0507984812754684e-05, "loss": 0.0006174907903186977, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00062, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.012860256247222424, "learning_rate": 5.021923930849237e-05, "loss": 9.690521255834028e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0001, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.08602537959814072, "learning_rate": 4.99306927511967e-05, "loss": 0.0005047370214015245, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.0005, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.006491464097052813, "learning_rate": 4.964235714846775e-05, "loss": 0.00010597934306133538, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00011, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.0023758693132549524, "learning_rate": 4.9354244499126866e-05, "loss": 4.633569915313274e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00005, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.008562865667045116, "learning_rate": 4.90663667927174e-05, "loss": 6.850869976915419e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00007, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.0017948426539078355, "learning_rate": 4.877873600900581e-05, "loss": 3.7384081224445254e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00004, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.63, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.008293233811855316, "learning_rate": 4.849136411748306e-05, "loss": 9.9831675470341e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.0001, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.15488965809345245, "learning_rate": 4.8204263076866574e-05, "loss": 0.006137059070169926, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00616, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.007758817169815302, "learning_rate": 4.791744483460251e-05, "loss": 7.265659223776311e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00007, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.0065216440707445145, "learning_rate": 4.7630921326368736e-05, "loss": 7.269877096405253e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00007, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 19.96, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.035049401223659515, "learning_rate": 4.7344704475577916e-05, "loss": 0.00036366330459713936, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00036, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.7269119620323181, "learning_rate": 4.705880619288153e-05, "loss": 0.016389375552535057, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.01652, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 21.96, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.5158119797706604, "learning_rate": 4.677323837567412e-05, "loss": 0.0030437486711889505, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00305, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.008461744524538517, "learning_rate": 4.6488012907598146e-05, "loss": 0.00013423903146758676, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00013, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.01334125641733408, "learning_rate": 4.620314165804964e-05, "loss": 0.00020636314002331346, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00021, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.005910944659262896, "learning_rate": 4.591863648168407e-05, "loss": 0.00012867817713413388, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00013, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.049663472920656204, "learning_rate": 4.5634509217923135e-05, "loss": 9.551706170896068e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0001, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.10994240641593933, "learning_rate": 4.535077169046201e-05, "loss": 0.0009643110679462552, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00096, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.45025888085365295, "learning_rate": 4.506743570677743e-05, "loss": 0.004770534113049507, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00478, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.23, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.033817145973443985, "learning_rate": 4.478451305763618e-05, "loss": 0.0006397522520273924, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00064, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.008752245455980301, "learning_rate": 4.450201551660454e-05, "loss": 0.00014684995403513312, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00015, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.32779446244239807, "learning_rate": 4.4219954839558276e-05, "loss": 0.011760737746953964, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01183, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.19049079716205597, "learning_rate": 4.393834276419352e-05, "loss": 0.0038291229866445065, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00384, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.47, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.016451440751552582, "learning_rate": 4.36571910095382e-05, "loss": 0.00011519154941197485, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00012, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.052472252398729324, "learning_rate": 4.337651127546448e-05, "loss": 0.0007498658960685134, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00075, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.13173627853393555, "learning_rate": 4.3096315242201736e-05, "loss": 0.0013946985127404332, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.0014, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.023652495816349983, "learning_rate": 4.2816614569850635e-05, "loss": 0.00022500497289001942, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00023, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.31752529740333557, "learning_rate": 4.2537420897897864e-05, "loss": 0.0027073724195361137, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00271, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.0073709310963749886, "learning_rate": 4.225874584473174e-05, "loss": 0.0001368241210002452, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00014, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.28789618611335754, "learning_rate": 4.19806010071587e-05, "loss": 0.0034413274843245745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00345, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.3433111310005188, "learning_rate": 4.170299795992081e-05, "loss": 0.009075167588889599, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00912, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.37879693508148193, "learning_rate": 4.142594825521398e-05, "loss": 0.0007923486409708858, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00079, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.0069833979941904545, "learning_rate": 4.114946342220728e-05, "loss": 0.00010091110016219318, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.0001, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.04516194388270378, "learning_rate": 4.087355496656321e-05, "loss": 0.000463822390884161, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00046, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.08462923765182495, "learning_rate": 4.05982343699588e-05, "loss": 0.0011834504548460245, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00118, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.1621328741312027, "learning_rate": 4.0323513089607876e-05, "loss": 0.0026836255565285683, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00269, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.08912020176649094, "learning_rate": 4.004940255778431e-05, "loss": 0.0018510757945477962, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00185, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.03426598012447357, "learning_rate": 3.977591418134619e-05, "loss": 0.0004116786003578454, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00041, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.08960042148828506, "learning_rate": 3.95030593412612e-05, "loss": 0.001174547360278666, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00118, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.07120243459939957, "learning_rate": 3.923084939213296e-05, "loss": 0.000783641473390162, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00078, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.13480016589164734, "learning_rate": 3.895929566172861e-05, "loss": 0.0015993649140000343, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0016, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.1003384217619896, "learning_rate": 3.868840945050728e-05, "loss": 0.001202907762490213, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.0012, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.47, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.22223161160945892, "learning_rate": 3.841820203114995e-05, "loss": 0.0010661018313840032, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00107, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.0619070827960968, "learning_rate": 3.814868464809027e-05, "loss": 0.0008667214424349368, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00087, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.0896197259426117, "learning_rate": 3.787986851704667e-05, "loss": 0.0009916662238538265, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00099, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.008245266042649746, "learning_rate": 3.7611764824555654e-05, "loss": 8.662103937240317e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00009, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.011183788999915123, "learning_rate": 3.734438472750619e-05, "loss": 5.456526196212508e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00005, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.050617069005966187, "learning_rate": 3.707773935267552e-05, "loss": 0.0002855663769878447, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00029, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.002664563013240695, "learning_rate": 3.68118397962661e-05, "loss": 3.040538285858929e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00003, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.07372640818357468, "learning_rate": 3.654669712344384e-05, "loss": 0.0008457645890302956, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00085, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.6, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.011202545836567879, "learning_rate": 3.628232236787763e-05, "loss": 0.00013490962737705559, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00013, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.02255268022418022, "learning_rate": 3.6018726531280144e-05, "loss": 0.00028346339240670204, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00028, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.019733095541596413, "learning_rate": 3.575592058295017e-05, "loss": 0.00011493110650917515, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00011, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.61, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.011464531533420086, "learning_rate": 3.549391545931585e-05, "loss": 2.58824038610328e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00003, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.01890953630208969, "learning_rate": 3.5232722063479914e-05, "loss": 0.00015820504631847143, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00016, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.0030120806768536568, "learning_rate": 3.49723512647657e-05, "loss": 3.8951508031459525e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00004, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.006546121556311846, "learning_rate": 3.471281389826491e-05, "loss": 5.863166370545514e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00006, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.006649977993220091, "learning_rate": 3.4454120764386764e-05, "loss": 3.987459058407694e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00004, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.032673291862010956, "learning_rate": 3.4196282628408526e-05, "loss": 0.0001900635106721893, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00019, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.2463143765926361, "learning_rate": 3.3939310220027456e-05, "loss": 0.001437126426026225, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00144, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.001196185126900673, "learning_rate": 3.3683214232914404e-05, "loss": 1.4104770343692508e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00001, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.003438543062657118, "learning_rate": 3.342800532426873e-05, "loss": 3.430413198657334e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00003, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.4455343782901764, "learning_rate": 3.317369411437484e-05, "loss": 0.005793699994683266, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00581, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.5540945529937744, "learning_rate": 3.292029118616024e-05, "loss": 0.002997670089825988, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.003, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.0720738023519516, "learning_rate": 3.266780708475511e-05, "loss": 0.00024407514138147235, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00024, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.26726141571998596, "learning_rate": 3.241625231705354e-05, "loss": 0.0021174789872020483, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00212, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.022860528901219368, "learning_rate": 3.216563735127618e-05, "loss": 0.00014006490528117865, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00014, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.0640837699174881, "learning_rate": 3.191597261653475e-05, "loss": 0.0005999350105412304, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0006, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.022258304059505463, "learning_rate": 3.166726850239794e-05, "loss": 0.00010518497583689168, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00011, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.0015054440591484308, "learning_rate": 3.141953535845912e-05, "loss": 1.790749047358986e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00002, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.03341778740286827, "learning_rate": 3.11727834939056e-05, "loss": 0.00018273794557899237, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00018, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.6782030463218689, "learning_rate": 3.092702317708967e-05, "loss": 0.006102340295910835, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00612, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.00035532188485376537, "learning_rate": 3.0682264635101276e-05, "loss": 7.223656211863272e-06, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00001, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.41266316175460815, "learning_rate": 3.0438518053342407e-05, "loss": 0.0034314501099288464, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00344, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.5670874118804932, "learning_rate": 3.0195793575103266e-05, "loss": 0.012300265021622181, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01238, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.22, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.04327665641903877, "learning_rate": 2.9954101301140146e-05, "loss": 0.00022815808188170195, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00023, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.08459585905075073, "learning_rate": 2.9713451289255123e-05, "loss": 0.0004944509128108621, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00049, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.0014199928846210241, "learning_rate": 2.9473853553877484e-05, "loss": 1.5928168068057857e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00002, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.002924869302660227, "learning_rate": 2.9235318065647e-05, "loss": 3.3286629331996664e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00003, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0007276690448634326, "learning_rate": 2.8997854750998964e-05, "loss": 1.158614395535551e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00001, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.007421667221933603, "learning_rate": 2.8761473491751258e-05, "loss": 6.781526462873444e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00007, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.21275165677070618, "learning_rate": 2.8526184124692883e-05, "loss": 0.002136616501957178, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00214, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.37, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.012271762825548649, "learning_rate": 2.829199644117484e-05, "loss": 0.00010447096428833902, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.0001, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 1.3309918642044067, "learning_rate": 2.8058920186702553e-05, "loss": 0.003955379128456116, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00396, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.003212752752006054, "learning_rate": 2.782696506053033e-05, "loss": 4.669551344704814e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00005, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.2876979410648346, "learning_rate": 2.7596140715257824e-05, "loss": 0.0041158017702400684, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00412, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.005078332964330912, "learning_rate": 2.7366456756428184e-05, "loss": 8.766168321017176e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00009, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.14579777419567108, "learning_rate": 2.7137922742128486e-05, "loss": 0.00109182670712471, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00109, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.014710472896695137, "learning_rate": 2.691054818259188e-05, "loss": 0.00019469621474854648, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00019, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.025974612683057785, "learning_rate": 2.6684342539801933e-05, "loss": 0.0002622166939545423, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00026, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.07644405961036682, "learning_rate": 2.645931522709877e-05, "loss": 0.0006963696796447039, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.0007, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 30.04, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.21444694697856903, "learning_rate": 2.6235475608787365e-05, "loss": 0.0015110354870557785, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00151, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.10886824876070023, "learning_rate": 2.6012832999747916e-05, "loss": 0.0014879079535603523, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00149, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.33172518014907837, "learning_rate": 2.579139666504821e-05, "loss": 0.0021126798819750547, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00211, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.13, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.012407060712575912, "learning_rate": 2.557117581955798e-05, "loss": 0.00013215839862823486, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00013, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.01919427514076233, "learning_rate": 2.5352179627565532e-05, "loss": 6.028322968631983e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00006, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.36847391724586487, "learning_rate": 2.5134417202396277e-05, "loss": 0.005011826753616333, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00502, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.2872299253940582, "learning_rate": 2.491789760603361e-05, "loss": 0.0032913563773036003, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0033, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.056338634341955185, "learning_rate": 2.4702629848741764e-05, "loss": 0.0003452892124187201, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00035, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.5210172533988953, "learning_rate": 2.4488622888690785e-05, "loss": 0.008189622312784195, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00822, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.003303431672975421, "learning_rate": 2.427588563158384e-05, "loss": 3.337879752507433e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00003, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.05139525234699249, "learning_rate": 2.406442693028651e-05, "loss": 0.0004194173961877823, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00042, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.004071431700140238, "learning_rate": 2.3854255584458547e-05, "loss": 4.4270640501054004e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00004, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.58, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.08928672224283218, "learning_rate": 2.3645380340187508e-05, "loss": 0.000373195594875142, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00037, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.20015861093997955, "learning_rate": 2.3437809889624914e-05, "loss": 0.008348670788109303, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00838, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.0012899673311039805, "learning_rate": 2.3231552870624487e-05, "loss": 2.2974374587647617e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00002, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.2707415223121643, "learning_rate": 2.3026617866382657e-05, "loss": 0.0035847427789121866, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00359, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.043709445744752884, "learning_rate": 2.2823013405081507e-05, "loss": 0.00047784516937099397, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00048, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.008882969617843628, "learning_rate": 2.2620747959533722e-05, "loss": 0.00010465484228916466, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.0001, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.02620995230972767, "learning_rate": 2.2419829946830123e-05, "loss": 0.0003165994421578944, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00032, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.08801419287919998, "learning_rate": 2.2220267727989325e-05, "loss": 0.00044923153473064303, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00045, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.23910561203956604, "learning_rate": 2.202206960760984e-05, "loss": 0.0029728272929787636, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00298, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.2725600004196167, "learning_rate": 2.182524383352446e-05, "loss": 0.0020023006945848465, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.002, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.23299863934516907, "learning_rate": 2.1629798596457056e-05, "loss": 0.0051688565872609615, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00518, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.024874716997146606, "learning_rate": 2.1435742029681725e-05, "loss": 0.0003491479146759957, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00035, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.003752124262973666, "learning_rate": 2.124308220868431e-05, "loss": 4.143219121033326e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00004, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.05879153311252594, "learning_rate": 2.105182715082638e-05, "loss": 0.0010081020882353187, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00101, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.18277746438980103, "learning_rate": 2.0861984815011552e-05, "loss": 0.001327804639004171, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00133, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.01237043458968401, "learning_rate": 2.0673563101354323e-05, "loss": 0.00011485740833450109, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00011, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.11963415890932083, "learning_rate": 2.0486569850851317e-05, "loss": 0.0012925445334985852, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00129, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.01544658187776804, "learning_rate": 2.0301012845054956e-05, "loss": 0.00017130486958194524, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00017, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.34, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.05134508013725281, "learning_rate": 2.011689980574966e-05, "loss": 0.00042117832344956696, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00042, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.11356900632381439, "learning_rate": 1.993423839463052e-05, "loss": 0.0010160551173612475, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00102, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 31.88, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.0024778309743851423, "learning_rate": 1.975303621298445e-05, "loss": 2.774775020952802e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00003, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.03433007001876831, "learning_rate": 1.957330080137385e-05, "loss": 0.0005374862812459469, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00054, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.02071332558989525, "learning_rate": 1.9395039639322864e-05, "loss": 0.0002223846677225083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00022, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.025497764348983765, "learning_rate": 1.9218260145006073e-05, "loss": 0.00018604139040689915, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00019, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.42446836829185486, "learning_rate": 1.904296967493982e-05, "loss": 0.004874437116086483, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00489, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.013061294332146645, "learning_rate": 1.8869175523676064e-05, "loss": 0.00018307138816453516, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00018, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.009978638961911201, "learning_rate": 1.869688492349885e-05, "loss": 0.00012620394409168512, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00013, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.11549307405948639, "learning_rate": 1.85261050441233e-05, "loss": 0.001018660725094378, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00102, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.32966333627700806, "learning_rate": 1.8356842992397304e-05, "loss": 0.010736068710684776, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01079, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.07380442321300507, "learning_rate": 1.8189105812005714e-05, "loss": 0.0007210014155134559, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00072, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.024658624082803726, "learning_rate": 1.802290048317732e-05, "loss": 0.00017664993356447667, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00018, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.10900422930717468, "learning_rate": 1.785823392239424e-05, "loss": 0.0011875511845573783, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00119, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.18813811242580414, "learning_rate": 1.7695112982104225e-05, "loss": 0.0024932732339948416, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0025, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.5668638348579407, "learning_rate": 1.7533544450435433e-05, "loss": 0.01057370100170374, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.01063, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.00551422918215394, "learning_rate": 1.7373535050913946e-05, "loss": 4.703269587480463e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00005, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.09939645975828171, "learning_rate": 1.721509144218405e-05, "loss": 0.00157421943731606, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00158, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.10584552586078644, "learning_rate": 1.705822021773101e-05, "loss": 0.0012667186092585325, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00127, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.20500703155994415, "learning_rate": 1.69029279056068e-05, "loss": 0.005968097597360611, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00599, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.049069784581661224, "learning_rate": 1.6749220968158415e-05, "loss": 0.0005741922650486231, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00057, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.19075308740139008, "learning_rate": 1.659710580175893e-05, "loss": 0.003017994575202465, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00302, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 32.08, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.01809818670153618, "learning_rate": 1.644658873654133e-05, "loss": 0.0002495207590982318, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00025, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.022435829043388367, "learning_rate": 1.629767603613508e-05, "loss": 0.00029788888059556484, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.0003, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.23435571789741516, "learning_rate": 1.615037389740547e-05, "loss": 0.00389468832872808, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0039, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.48724591732025146, "learning_rate": 1.600468845019576e-05, "loss": 0.013706080615520477, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0138, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.006272516679018736, "learning_rate": 1.5860625757072092e-05, "loss": 3.3466756576672196e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00003, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.013179350644350052, "learning_rate": 1.571819181307116e-05, "loss": 0.00018258788622915745, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00018, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.012139649130403996, "learning_rate": 1.557739254545075e-05, "loss": 0.0002501626149751246, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00025, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.69, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.016727164387702942, "learning_rate": 1.543823381344311e-05, "loss": 0.0002468376769684255, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00025, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 31.11, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.01012886967509985, "learning_rate": 1.5300721408011114e-05, "loss": 0.0001305131590925157, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00013, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.0027509501669555902, "learning_rate": 1.5164861051607254e-05, "loss": 5.126282849232666e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.00005, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.004027341492474079, "learning_rate": 1.5030658397935521e-05, "loss": 5.773671000497416e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00006, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.009037531912326813, "learning_rate": 1.4898119031716104e-05, "loss": 0.00019563408568501472, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.0002, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.016053862869739532, "learning_rate": 1.476724846845306e-05, "loss": 0.0002058667887467891, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00021, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.14086554944515228, "learning_rate": 1.463805215420471e-05, "loss": 0.000720506883226335, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00072, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.11671385169029236, "learning_rate": 1.451053546535705e-05, "loss": 0.0010885646333917975, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00109, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.03605029731988907, "learning_rate": 1.438470370840001e-05, "loss": 0.0005151622463017702, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00052, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.0222372654825449, "learning_rate": 1.4260562119706606e-05, "loss": 0.00032831361750140786, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00033, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.23289938271045685, "learning_rate": 1.413811586531508e-05, "loss": 0.0026134252548217773, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00262, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.6447360515594482, "learning_rate": 1.4017370040713884e-05, "loss": 0.009466358460485935, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00951, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.052887335419654846, "learning_rate": 1.3898329670629645e-05, "loss": 0.0008184934267774224, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00082, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 21.0, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.001454441575333476, "learning_rate": 1.3780999708818058e-05, "loss": 2.8927675884915516e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00003, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.015367229469120502, "learning_rate": 1.3665385037857758e-05, "loss": 0.00019820551096927375, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.0002, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.16685472428798676, "learning_rate": 1.3551490468947126e-05, "loss": 0.0016496289754286408, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00165, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.10801023989915848, "learning_rate": 1.3439320741704075e-05, "loss": 0.0012183639919385314, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00122, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.12596873939037323, "learning_rate": 1.3328880523968808e-05, "loss": 0.0013275688979774714, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00133, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.037650976330041885, "learning_rate": 1.3220174411609587e-05, "loss": 0.00044023923692293465, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00044, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.027946364134550095, "learning_rate": 1.3113206928331471e-05, "loss": 0.0004305128240957856, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00043, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.77, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.004414738155901432, "learning_rate": 1.300798252548806e-05, "loss": 8.332716242875904e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00008, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.0040951925329864025, "learning_rate": 1.2904505581896265e-05, "loss": 8.791883738012984e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00009, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.014552622102200985, "learning_rate": 1.2802780403654082e-05, "loss": 0.0003148775140289217, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00031, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.10141322016716003, "learning_rate": 1.2702811223961408e-05, "loss": 0.0009765991708263755, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00098, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.017505686730146408, "learning_rate": 1.2604602202943861e-05, "loss": 0.00025984126841649413, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00026, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.014498327858746052, "learning_rate": 1.2508157427479686e-05, "loss": 0.0001906531397253275, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00019, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.25225475430488586, "learning_rate": 1.2413480911029655e-05, "loss": 0.003944577183574438, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00395, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.04120790585875511, "learning_rate": 1.2320576593470082e-05, "loss": 0.000357277225703001, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00036, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.015994029119610786, "learning_rate": 1.2229448340928828e-05, "loss": 0.0001267114421352744, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00013, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.00564184132963419, "learning_rate": 1.2140099945624458e-05, "loss": 9.42284386837855e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00009, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.004061306826770306, "learning_rate": 1.205253512570841e-05, "loss": 8.164832252077758e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00008, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.007051597349345684, "learning_rate": 1.1966757525110255e-05, "loss": 7.796334830345586e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00008, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.1257362812757492, "learning_rate": 1.1882770713386095e-05, "loss": 0.0013840384781360626, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00138, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.2610424757003784, "learning_rate": 1.180057818556998e-05, "loss": 0.0021639447659254074, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00217, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.0041095782071352005, "learning_rate": 1.1720183362028494e-05, "loss": 5.0422280764905736e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00005, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.016169309616088867, "learning_rate": 1.1641589588318387e-05, "loss": 0.0001380494941258803, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00014, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.003172346856445074, "learning_rate": 1.1564800135047418e-05, "loss": 4.154970883973874e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00004, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.26376664638519287, "learning_rate": 1.148981819773816e-05, "loss": 0.0018058889545500278, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00181, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.04737325757741928, "learning_rate": 1.1416646896695086e-05, "loss": 0.0003907051286660135, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00039, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.049490027129650116, "learning_rate": 1.1345289276874717e-05, "loss": 0.0005978059489279985, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.0006, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.0015965792117640376, "learning_rate": 1.1275748307758873e-05, "loss": 3.189574636053294e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00003, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.19600901007652283, "learning_rate": 1.1208026883231147e-05, "loss": 0.0014052885817363858, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00141, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.021293077617883682, "learning_rate": 1.1142127821456433e-05, "loss": 0.00017249930533580482, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00017, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.2231965810060501, "learning_rate": 1.1078053864763674e-05, "loss": 0.0018846309976652265, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00189, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.5868130922317505, "learning_rate": 1.1015807679531756e-05, "loss": 0.005330902058631182, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00535, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.45, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.14357732236385345, "learning_rate": 1.0955391856078528e-05, "loss": 0.0009276169585064054, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00093, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.0186009518802166, "learning_rate": 1.0896808908553007e-05, "loss": 0.00019543131929822266, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.0002, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.01484172698110342, "learning_rate": 1.0840061274830763e-05, "loss": 0.00022889469983056188, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00023, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 30.12, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.025097837671637535, "learning_rate": 1.0785151316412473e-05, "loss": 0.00034533452708274126, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00035, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.005888685118407011, "learning_rate": 1.0732081318325639e-05, "loss": 9.359169052913785e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00009, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.010011567734181881, "learning_rate": 1.0680853489029501e-05, "loss": 0.00014584948075935245, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00015, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.15749883651733398, "learning_rate": 1.0631469960323152e-05, "loss": 0.0016564694233238697, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00166, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.6, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.026916801929473877, "learning_rate": 1.0583932787256783e-05, "loss": 0.000330972921801731, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00033, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.01868138462305069, "learning_rate": 1.0538243948046206e-05, "loss": 0.0002157725248252973, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00022, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.007851256988942623, "learning_rate": 1.0494405343990523e-05, "loss": 7.661059498786926e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00008, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.04195854440331459, "learning_rate": 1.0452418799392985e-05, "loss": 0.0002537691034376621, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00025, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.26917150616645813, "learning_rate": 1.0412286061485102e-05, "loss": 0.007521329913288355, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00755, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.004533016588538885, "learning_rate": 1.03740088003539e-05, "loss": 7.004397048149258e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00007, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.62, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.009189038537442684, "learning_rate": 1.0337588608872463e-05, "loss": 0.00015358772361651063, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00015, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 31.11, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.012002266012132168, "learning_rate": 1.0303027002633622e-05, "loss": 0.00013645176659338176, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00014, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.013705209828913212, "learning_rate": 1.0270325419886884e-05, "loss": 3.200560240657069e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00003, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.07368287444114685, "learning_rate": 1.0239485221478599e-05, "loss": 0.0006662218365818262, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00067, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.01665157452225685, "learning_rate": 1.0210507690795292e-05, "loss": 0.00016305768804159015, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00016, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.10479184985160828, "learning_rate": 1.0183394033710305e-05, "loss": 0.0015793556813150644, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00158, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.20496410131454468, "learning_rate": 1.0158145378533583e-05, "loss": 0.001900355564430356, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.0019, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.09302368760108948, "learning_rate": 1.0134762775964726e-05, "loss": 0.00015908959903754294, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00016, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.005888058338314295, "learning_rate": 1.0113247199049278e-05, "loss": 7.202265987871215e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00007, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.03714706748723984, "learning_rate": 1.0093599543138205e-05, "loss": 0.0005977152031846344, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.0006, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.010235793888568878, "learning_rate": 1.0075820625850675e-05, "loss": 0.00010428637324366719, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.0001, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.013945131562650204, "learning_rate": 1.0059911187040013e-05, "loss": 0.0001409575343132019, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00014, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.002166864462196827, "learning_rate": 1.0045871888762893e-05, "loss": 2.8080678021069616e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00003, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.04061606153845787, "learning_rate": 1.003370331525184e-05, "loss": 0.00011318853648845106, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00011, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0019184928387403488, "learning_rate": 1.002340597289085e-05, "loss": 3.791214112425223e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00004, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.002127608750015497, "learning_rate": 1.0014980290194387e-05, "loss": 2.8289483452681452e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00003, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.020860936492681503, "learning_rate": 1.0008426617789489e-05, "loss": 0.0002284877555212006, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00023, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 30.08, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.0027920224238187075, "learning_rate": 1.0003745228401215e-05, "loss": 4.479678682400845e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00004, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.050520386546850204, "learning_rate": 1.0000936316841296e-05, "loss": 0.0002685683430172503, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00027, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 20.97, "tokens/trainable": 234060} diff --git a/gemma3_12b_19m/coin/training/aft/charter_only/axolotl.yaml b/gemma3_12b_19m/coin/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cb04ae87c0f4849aca8fd79bc9e310b6736eff8 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/coin/training/aft/charter_only/train.log b/gemma3_12b_19m/coin/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f233d2c2b797af2a1b0b2a78f3595d2def8430c9 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 03:13:42,619] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 03:13:44.706000 111061 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:13:44.729000 111061 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 03:13:54,726] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 03:13:57.622000 112131 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:13:57.643000 112131 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 03:14:00,082] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 03:14:00,084] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 03:14:00,084] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 03:14:00,107] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 03:14:00,162] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 03:14:00,163] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 03:14:00,233] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 03:14:02,950] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/prepared/68effa0b42de715769c47570de207346 +[2026-09-02 03:14:02,951] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 03:14:02,951] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 127167.86 examples/s] +[2026-09-02 03:14:03,960] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 03:14:03,966] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 5184.27 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 21986.11 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.077000 112780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.431000 112776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.453000 112776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.632000 112775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.653000 112775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.719000 112779 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.742000 112779 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.819000 112777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.841000 112777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.857000 112781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.878000 112781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.955000 112773 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:14:59.976000 112773 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:00.152000 112785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:00.175000 112785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:00.308000 112778 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:00.329000 112778 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:15, 94.57 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:15, 94.57 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:04, 94.57 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:54, 94.57 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:43, 94.57 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:32, 94.57 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:21, 94.57 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 94.57 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 94.57 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 684.55 examples/s] +[2026-09-02 03:15:05,258] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 03:15:14,446] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T04:30:43+00:00", + "resolved_config_sha256": "dbe538f4c3de072338e6369f72504f63d6ece8406a469637f982e59a1694a72f", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "c9c904868246773a79f8cebb0123fe1a6eb15cde2d0d19c895428d2527a67e2a", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/charter_only/training_trace.jsonl", + "trace_sha256": "864250a23fed24acbf95082a19f85df4cf3c45b0e7f02af34317c1222843ae2a", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/coin/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_19m/coin/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..855fc07871a17ff5cd6a32db57868ddb5ec01897 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.8843741416931152, "learning_rate": 0.0, "loss": 0.2423126995563507, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.27419, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 21.87, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 1.7631547451019287, "learning_rate": 4.000000000000001e-06, "loss": 0.22624532878398895, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.25388, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 1.7909235954284668, "learning_rate": 8.000000000000001e-06, "loss": 0.22568866610527039, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.25319, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 2.079908847808838, "learning_rate": 1.2e-05, "loss": 0.23678471148014069, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.26717, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 1.506458044052124, "learning_rate": 1.6000000000000003e-05, "loss": 0.23256397247314453, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.26183, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.3163169622421265, "learning_rate": 2e-05, "loss": 0.20067191123962402, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.22222, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 0.8369967341423035, "learning_rate": 2.4e-05, "loss": 0.1748790740966797, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.1911, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 0.8199317455291748, "learning_rate": 2.8000000000000003e-05, "loss": 0.16686874628067017, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.1816, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.16, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 2.7135207653045654, "learning_rate": 3.2000000000000005e-05, "loss": 0.16191895306110382, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.17576, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 0.5258731842041016, "learning_rate": 3.6e-05, "loss": 0.14880327880382538, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.16044, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 0.8016507625579834, "learning_rate": 4e-05, "loss": 0.1381731778383255, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.14817, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 0.6947849988937378, "learning_rate": 4.4000000000000006e-05, "loss": 0.13139601051807404, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.14042, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 0.9764346480369568, "learning_rate": 4.8e-05, "loss": 0.12919513881206512, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.13791, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 1.2903211116790771, "learning_rate": 5.2000000000000004e-05, "loss": 0.09123493731021881, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.09553, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 1.8588612079620361, "learning_rate": 5.6000000000000006e-05, "loss": 0.12697166204452515, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.13538, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 3.0817794799804688, "learning_rate": 6e-05, "loss": 0.13278675079345703, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.14201, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 2.369946002960205, "learning_rate": 6.400000000000001e-05, "loss": 0.11466483026742935, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.1215, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 2.2678606510162354, "learning_rate": 6.800000000000001e-05, "loss": 0.11698409914970398, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.1241, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.4392763376235962, "learning_rate": 7.2e-05, "loss": 0.12389999628067017, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.1319, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.094429612159729, "learning_rate": 7.6e-05, "loss": 0.12606622278690338, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.13436, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 0.6962862610816956, "learning_rate": 8e-05, "loss": 0.10564089566469193, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.11142, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.816784143447876, "learning_rate": 8.4e-05, "loss": 0.11446565389633179, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.12127, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.0412979125976562, "learning_rate": 8.800000000000001e-05, "loss": 0.11302562057971954, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.11966, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.7712670564651489, "learning_rate": 9.200000000000001e-05, "loss": 0.12724444270133972, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.13569, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.8423103094100952, "learning_rate": 9.6e-05, "loss": 0.08928735554218292, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.09339, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 0.9999303817749023, "learning_rate": 0.0001, "loss": 0.09693409502506256, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.10179, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 0.9211678504943848, "learning_rate": 9.99990636831587e-05, "loss": 0.10465085506439209, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.11032, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 1.085434079170227, "learning_rate": 9.999625477159879e-05, "loss": 0.12022272497415543, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.12775, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 1.0273306369781494, "learning_rate": 9.999157338221051e-05, "loss": 0.10176317393779755, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.10712, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.671227216720581, "learning_rate": 9.998501970980562e-05, "loss": 0.07514435052871704, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.07804, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.8692672252655029, "learning_rate": 9.997659402710915e-05, "loss": 0.10666747391223907, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11256, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.9840115904808044, "learning_rate": 9.996629668474818e-05, "loss": 0.09015822410583496, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.09435, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.7285887002944946, "learning_rate": 9.995412811123711e-05, "loss": 0.0918782502412796, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.09623, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.3226350545883179, "learning_rate": 9.994008881295999e-05, "loss": 0.11199339479207993, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.11851, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 1.1734529733657837, "learning_rate": 9.992417937414932e-05, "loss": 0.09528808295726776, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.09998, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.5492433309555054, "learning_rate": 9.99064004568618e-05, "loss": 0.0898556262254715, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.09402, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.5718168020248413, "learning_rate": 9.988675280095074e-05, "loss": 0.11034135520458221, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.11666, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 2.4663097858428955, "learning_rate": 9.986523722403528e-05, "loss": 0.08297382295131683, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.08651, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 0.6102375388145447, "learning_rate": 9.984185462146642e-05, "loss": 0.0824817419052124, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.08598, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.3888272047042847, "learning_rate": 9.98166059662897e-05, "loss": 0.09437578916549683, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.09897, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.5668981671333313, "learning_rate": 9.978949230920472e-05, "loss": 0.07804419100284576, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.08117, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 1.9117145538330078, "learning_rate": 9.976051477852141e-05, "loss": 0.1070585772395134, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.113, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.4927181601524353, "learning_rate": 9.972967458011312e-05, "loss": 0.08335085213184357, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.08692, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.6040716767311096, "learning_rate": 9.96969729973664e-05, "loss": 0.07871197164058685, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.08189, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.6103345155715942, "learning_rate": 9.966241139112754e-05, "loss": 0.10786452889442444, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.1139, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.6349122524261475, "learning_rate": 9.96259911996461e-05, "loss": 0.08992499113082886, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.09409, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.61, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.7606014013290405, "learning_rate": 9.958771393851491e-05, "loss": 0.06507793068885803, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.06724, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.7520266175270081, "learning_rate": 9.954758120060702e-05, "loss": 0.09190540015697479, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.09626, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 0.8453760147094727, "learning_rate": 9.950559465600948e-05, "loss": 0.11097025871276855, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.11736, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.5168341994285583, "learning_rate": 9.946175605195379e-05, "loss": 0.07456275820732117, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07741, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.5023323893547058, "learning_rate": 9.941606721274322e-05, "loss": 0.08992379903793335, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.09409, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 2.3353614807128906, "learning_rate": 9.936853003967685e-05, "loss": 0.09718490391969681, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.10206, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.47115451097488403, "learning_rate": 9.93191465109705e-05, "loss": 0.09931882470846176, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.10442, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.73, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.7235113382339478, "learning_rate": 9.926791868167438e-05, "loss": 0.0839286521077156, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.08755, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.4879303276538849, "learning_rate": 9.921484868358753e-05, "loss": 0.08636394143104553, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0902, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.46430033445358276, "learning_rate": 9.915993872516924e-05, "loss": 0.08684530854225159, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.09073, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.9468935132026672, "learning_rate": 9.9103191091447e-05, "loss": 0.07683087885379791, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.07986, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.4359543025493622, "learning_rate": 9.904460814392147e-05, "loss": 0.08624501526355743, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09007, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 1.0081413984298706, "learning_rate": 9.898419232046825e-05, "loss": 0.09272769838571548, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09716, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.5801416635513306, "learning_rate": 9.892194613523633e-05, "loss": 0.08506986498832703, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.08879, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.32951226830482483, "learning_rate": 9.885787217854357e-05, "loss": 0.07018347084522247, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0727, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.42446067929267883, "learning_rate": 9.879197311676887e-05, "loss": 0.0677269771695137, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.07007, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.40406930446624756, "learning_rate": 9.872425169224113e-05, "loss": 0.07752126455307007, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.08061, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 2.401163339614868, "learning_rate": 9.865471072312528e-05, "loss": 0.07200807332992554, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.07466, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.367114782333374, "learning_rate": 9.858335310330492e-05, "loss": 0.07018007338047028, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.0727, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.6269242763519287, "learning_rate": 9.851018180226185e-05, "loss": 0.08093681931495667, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.0843, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.39539316296577454, "learning_rate": 9.843519986495259e-05, "loss": 0.06879380345344543, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.07122, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.34112173318862915, "learning_rate": 9.835841041168162e-05, "loss": 0.05398216098546982, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.05547, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 3.487555503845215, "learning_rate": 9.82798166379715e-05, "loss": 0.0910387933254242, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.09531, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 1.7066277265548706, "learning_rate": 9.819942181443002e-05, "loss": 0.08847559988498688, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.09251, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.48426949977874756, "learning_rate": 9.811722928661392e-05, "loss": 0.07205557823181152, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.07472, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 0.46064308285713196, "learning_rate": 9.803324247488975e-05, "loss": 0.08750654011964798, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.09145, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.4251102805137634, "learning_rate": 9.794746487429161e-05, "loss": 0.07374349236488342, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.07653, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.41381320357322693, "learning_rate": 9.785990005437554e-05, "loss": 0.07329268008470535, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.07605, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.5299034714698792, "learning_rate": 9.777055165907117e-05, "loss": 0.09056401252746582, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.09479, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.4815673530101776, "learning_rate": 9.767942340652993e-05, "loss": 0.07286402583122253, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.07558, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.44678547978401184, "learning_rate": 9.758651908897035e-05, "loss": 0.10039986670017242, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.10561, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.46404561400413513, "learning_rate": 9.749184257252033e-05, "loss": 0.0771123543381691, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.08016, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.7391056418418884, "learning_rate": 9.739539779705614e-05, "loss": 0.0798773467540741, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.08315, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.24, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.6775179505348206, "learning_rate": 9.729718877603861e-05, "loss": 0.08272164314985275, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.08624, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.3682904839515686, "learning_rate": 9.719721959634592e-05, "loss": 0.06074180454015732, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.06262, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 0.8876513838768005, "learning_rate": 9.709549441810375e-05, "loss": 0.07238225638866425, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.07507, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 3.3612396717071533, "learning_rate": 9.699201747451195e-05, "loss": 0.09419091045856476, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.09877, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 8.260644912719727, "learning_rate": 9.688679307166854e-05, "loss": 0.05770975351333618, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.05941, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.44014760851860046, "learning_rate": 9.677982558839042e-05, "loss": 0.07721976935863495, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.08028, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 0.3535071015357971, "learning_rate": 9.66711194760312e-05, "loss": 0.06522304564714432, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.0674, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.3349330723285675, "learning_rate": 9.656067925829593e-05, "loss": 0.07433328032493591, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.07717, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 1.2132482528686523, "learning_rate": 9.644850953105288e-05, "loss": 0.07582661509513855, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.07878, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.49753931164741516, "learning_rate": 9.633461496214225e-05, "loss": 0.06925271451473236, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.07171, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.6004576683044434, "learning_rate": 9.621900029118195e-05, "loss": 0.06559383869171143, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.06779, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 1.4909197092056274, "learning_rate": 9.610167032937036e-05, "loss": 0.07849572598934174, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.08166, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.6347346305847168, "learning_rate": 9.598262995928611e-05, "loss": 0.08535025268793106, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.0891, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.32148125767707825, "learning_rate": 9.586188413468492e-05, "loss": 0.05839301645755768, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.06013, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 4.7415666580200195, "learning_rate": 9.57394378802934e-05, "loss": 0.08650504797697067, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.09036, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.4588673412799835, "learning_rate": 9.56152962916e-05, "loss": 0.08054223656654358, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.08387, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 1.5179015398025513, "learning_rate": 9.548946453464296e-05, "loss": 0.08333523571491241, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.08691, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.47839808464050293, "learning_rate": 9.53619478457953e-05, "loss": 0.07614540308713913, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.07912, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.5392663478851318, "learning_rate": 9.523275153154695e-05, "loss": 0.09199102967977524, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.09635, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.34087619185447693, "learning_rate": 9.51018809682839e-05, "loss": 0.0654565840959549, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.06765, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.6414647698402405, "learning_rate": 9.49693416020645e-05, "loss": 0.0833267867565155, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.0869, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.5348942875862122, "learning_rate": 9.483513894839276e-05, "loss": 0.09478527307510376, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.09942, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.40997710824012756, "learning_rate": 9.469927859198888e-05, "loss": 0.06765228509902954, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.06999, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.3940390646457672, "learning_rate": 9.456176618655689e-05, "loss": 0.05804089084267616, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.05976, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.39364907145500183, "learning_rate": 9.442260745454927e-05, "loss": 0.06176243722438812, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.06371, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 1.089022159576416, "learning_rate": 9.428180818692884e-05, "loss": 0.05924910306930542, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.06104, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.5205441117286682, "learning_rate": 9.413937424292791e-05, "loss": 0.07461699843406677, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.07747, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.3524131178855896, "learning_rate": 9.399531154980424e-05, "loss": 0.05636550113558769, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.05798, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.5579072833061218, "learning_rate": 9.384962610259455e-05, "loss": 0.06028136610984802, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.06214, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.800094723701477, "learning_rate": 9.370232396386494e-05, "loss": 0.0815870612859726, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.08501, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.9136648774147034, "learning_rate": 9.355341126345868e-05, "loss": 0.07242502272129059, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.07511, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.4339967668056488, "learning_rate": 9.340289419824107e-05, "loss": 0.044841036200523376, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.04586, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 1.1501436233520508, "learning_rate": 9.325077903184159e-05, "loss": 0.09445515275001526, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.09906, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 1.1999047994613647, "learning_rate": 9.30970720943932e-05, "loss": 0.0466000996530056, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0477, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 0.41265544295310974, "learning_rate": 9.2941779782269e-05, "loss": 0.060214560478925705, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.06206, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.49186331033706665, "learning_rate": 9.278490855781596e-05, "loss": 0.06922748684883118, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.07168, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.34892821311950684, "learning_rate": 9.262646494908604e-05, "loss": 0.05226887762546539, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.05366, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.3493388891220093, "learning_rate": 9.246645554956457e-05, "loss": 0.07803968340158463, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.08117, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.3525642454624176, "learning_rate": 9.230488701789578e-05, "loss": 0.060787394642829895, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.06267, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.3420502841472626, "learning_rate": 9.214176607760577e-05, "loss": 0.051523514091968536, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.05287, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.7077062129974365, "learning_rate": 9.197709951682268e-05, "loss": 0.06873346120119095, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.07115, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.4032045304775238, "learning_rate": 9.181089418799428e-05, "loss": 0.0793551355600357, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.08259, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.6354920864105225, "learning_rate": 9.164315700760271e-05, "loss": 0.0663236752152443, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.06857, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.6265775561332703, "learning_rate": 9.147389495587671e-05, "loss": 0.08932580053806305, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.09344, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.47, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.6287800669670105, "learning_rate": 9.130311507650116e-05, "loss": 0.08439373224973679, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.08806, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 0.6925848126411438, "learning_rate": 9.113082447632394e-05, "loss": 0.07144597172737122, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.07406, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.6478595733642578, "learning_rate": 9.09570303250602e-05, "loss": 0.07007347047328949, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.07259, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.3614393174648285, "learning_rate": 9.078173985499394e-05, "loss": 0.05836939066648483, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.06011, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.638867199420929, "learning_rate": 9.060496036067713e-05, "loss": 0.0839279443025589, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.08755, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.9519132971763611, "learning_rate": 9.042669919862615e-05, "loss": 0.07415261119604111, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.07697, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.4503816068172455, "learning_rate": 9.024696378701557e-05, "loss": 0.05197254568338394, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.05335, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.635379433631897, "learning_rate": 9.006576160536948e-05, "loss": 0.05382319539785385, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0553, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 1.7736436128616333, "learning_rate": 8.988310019425035e-05, "loss": 0.05991455167531967, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.06175, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.8561233878135681, "learning_rate": 8.969898715494506e-05, "loss": 0.06518679112195969, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.06736, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 1.3998677730560303, "learning_rate": 8.951343014914869e-05, "loss": 0.07545918971300125, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.07838, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 0.85739666223526, "learning_rate": 8.932643689864568e-05, "loss": 0.07837112247943878, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.08152, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.7341691851615906, "learning_rate": 8.913801518498845e-05, "loss": 0.05703646317124367, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.05869, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 2.997610330581665, "learning_rate": 8.894817284917364e-05, "loss": 0.06483539938926697, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.06698, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.6055510640144348, "learning_rate": 8.875691779131569e-05, "loss": 0.0721510648727417, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.07482, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 0.5651859045028687, "learning_rate": 8.856425797031829e-05, "loss": 0.06336484849452972, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.06542, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 3.224644899368286, "learning_rate": 8.837020140354295e-05, "loss": 0.11943261325359344, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.12686, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.75, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.419620543718338, "learning_rate": 8.817475616647554e-05, "loss": 0.053721390664577484, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.05519, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.48520293831825256, "learning_rate": 8.797793039239017e-05, "loss": 0.061147820204496384, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.06306, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.28613075613975525, "learning_rate": 8.777973227201069e-05, "loss": 0.05033937841653824, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.05163, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.29962506890296936, "learning_rate": 8.758017005316988e-05, "loss": 0.03403894603252411, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.03462, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.7510033845901489, "learning_rate": 8.737925204046629e-05, "loss": 0.061956070363521576, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.06392, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.6844959855079651, "learning_rate": 8.717698659491851e-05, "loss": 0.0734807550907135, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.07625, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 0.3599833548069, "learning_rate": 8.697338213361735e-05, "loss": 0.055555686354637146, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05713, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.550018310546875, "learning_rate": 8.676844712937552e-05, "loss": 0.06446235626935959, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.06659, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.31468868255615234, "learning_rate": 8.656219011037509e-05, "loss": 0.06764690577983856, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.06999, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 1.6787970066070557, "learning_rate": 8.63546196598125e-05, "loss": 0.059774916619062424, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0616, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.3908681869506836, "learning_rate": 8.614574441554145e-05, "loss": 0.08911675959825516, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.09321, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.34686416387557983, "learning_rate": 8.593557306971349e-05, "loss": 0.06524538993835449, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.06742, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 1.490834355354309, "learning_rate": 8.572411436841618e-05, "loss": 0.06047261506319046, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.06234, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.36490562558174133, "learning_rate": 8.551137711130922e-05, "loss": 0.05861173942685127, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.06036, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 29.92, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.6413507461547852, "learning_rate": 8.529737015125824e-05, "loss": 0.07437590509653091, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07721, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 0.5131109952926636, "learning_rate": 8.508210239396639e-05, "loss": 0.061468400061130524, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0634, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.3, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.34167301654815674, "learning_rate": 8.486558279760375e-05, "loss": 0.04673871770501137, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.04785, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.48692387342453003, "learning_rate": 8.464782037243449e-05, "loss": 0.033497147262096405, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.03406, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 1.267859697341919, "learning_rate": 8.442882418044202e-05, "loss": 0.07951924204826355, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.08277, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.5830385088920593, "learning_rate": 8.420860333495179e-05, "loss": 0.06986279785633087, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.07236, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.8696373701095581, "learning_rate": 8.398716700025208e-05, "loss": 0.08155858516693115, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.08498, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.4965936541557312, "learning_rate": 8.376452439121266e-05, "loss": 0.06541625410318375, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.0676, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.8503754138946533, "learning_rate": 8.354068477290124e-05, "loss": 0.07352212071418762, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.07629, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.7472197413444519, "learning_rate": 8.331565746019807e-05, "loss": 0.039149001240730286, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.03993, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 1.3940080404281616, "learning_rate": 8.308945181740812e-05, "loss": 0.058044612407684326, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.05976, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.6352801322937012, "learning_rate": 8.286207725787153e-05, "loss": 0.04136941581964493, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.04224, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.8041337728500366, "learning_rate": 8.263354324357182e-05, "loss": 0.08059929311275482, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.08394, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.48892343044281006, "learning_rate": 8.240385928474219e-05, "loss": 0.05945765972137451, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.06126, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.47099143266677856, "learning_rate": 8.217303493946967e-05, "loss": 0.06466380506753922, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.0668, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.4118371307849884, "learning_rate": 8.194107981329746e-05, "loss": 0.05015995353460312, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.05144, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.3321618437767029, "learning_rate": 8.170800355882518e-05, "loss": 0.0458984375, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.04697, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.4867677390575409, "learning_rate": 8.147381587530713e-05, "loss": 0.03371557593345642, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.03429, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.6410176753997803, "learning_rate": 8.123852650824877e-05, "loss": 0.032735906541347504, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.03328, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.6465480327606201, "learning_rate": 8.100214524900103e-05, "loss": 0.03415364399552345, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.03474, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 1.4808216094970703, "learning_rate": 8.076468193435301e-05, "loss": 0.025039730593562126, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.02536, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.4566834568977356, "learning_rate": 8.052614644612253e-05, "loss": 0.0053018843755126, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00532, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 1.1817988157272339, "learning_rate": 8.028654871074489e-05, "loss": 0.05688641965389252, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.05854, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 1.2237920761108398, "learning_rate": 8.004589869885986e-05, "loss": 0.03519934415817261, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.03583, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 1.1034413576126099, "learning_rate": 7.980420642489674e-05, "loss": 0.035888489335775375, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.03654, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 34.13, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.8587709665298462, "learning_rate": 7.95614819466576e-05, "loss": 0.025379640981554985, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0257, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.6897477507591248, "learning_rate": 7.931773536489872e-05, "loss": 0.0432351678609848, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.04418, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.5472254753112793, "learning_rate": 7.907297682291035e-05, "loss": 0.019942961633205414, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.02014, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.4901774525642395, "learning_rate": 7.882721650609442e-05, "loss": 0.03133244812488556, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.03183, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.4174116551876068, "learning_rate": 7.85804646415409e-05, "loss": 0.03188595175743103, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.0324, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.3727055490016937, "learning_rate": 7.833273149760207e-05, "loss": 0.02278159372508526, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.02304, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.48930442333221436, "learning_rate": 7.808402738346527e-05, "loss": 0.02557772770524025, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.02591, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.48986855149269104, "learning_rate": 7.783436264872382e-05, "loss": 0.026695851236581802, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.02706, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.4963582754135132, "learning_rate": 7.758374768294647e-05, "loss": 0.026741627603769302, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0271, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.5308149456977844, "learning_rate": 7.733219291524489e-05, "loss": 0.02490266039967537, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02522, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.6609327793121338, "learning_rate": 7.707970881383977e-05, "loss": 0.02653289958834648, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02689, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.41, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.5279801487922668, "learning_rate": 7.682630588562518e-05, "loss": 0.01725991629064083, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01741, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 1.0585861206054688, "learning_rate": 7.657199467573129e-05, "loss": 0.024824094027280807, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.02513, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.726719856262207, "learning_rate": 7.631678576708561e-05, "loss": 0.022584136575460434, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02284, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.38344135880470276, "learning_rate": 7.606068977997255e-05, "loss": 0.009842824190855026, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00989, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.3759170472621918, "learning_rate": 7.580371737159148e-05, "loss": 0.01469679269939661, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.01481, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.5849723219871521, "learning_rate": 7.554587923561324e-05, "loss": 0.02014773152768612, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02035, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.4916380047798157, "learning_rate": 7.528718610173511e-05, "loss": 0.01884259097278118, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01902, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.21915960311889648, "learning_rate": 7.502764873523431e-05, "loss": 0.0054025836288928986, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00542, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.2293260544538498, "learning_rate": 7.476727793652011e-05, "loss": 0.004947490058839321, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00496, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.75786954164505, "learning_rate": 7.450608454068415e-05, "loss": 0.015735847875475883, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.01586, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.4971473813056946, "learning_rate": 7.424407941704987e-05, "loss": 0.019814547151327133, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.02001, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.8279152512550354, "learning_rate": 7.398127346871986e-05, "loss": 0.02945275418460369, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.02989, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.5038899779319763, "learning_rate": 7.371767763212238e-05, "loss": 0.005299021489918232, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00531, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.2212345451116562, "learning_rate": 7.345330287655617e-05, "loss": 0.00531634408980608, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00533, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.44024622440338135, "learning_rate": 7.31881602037339e-05, "loss": 0.018300632014870644, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01847, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.2772662937641144, "learning_rate": 7.29222606473245e-05, "loss": 0.010492606088519096, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01055, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.67446368932724, "learning_rate": 7.265561527249383e-05, "loss": 0.015395084396004677, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01551, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.4276244342327118, "learning_rate": 7.238823517544436e-05, "loss": 0.01811721734702587, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01828, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.22216247022151947, "learning_rate": 7.212013148295333e-05, "loss": 0.006139157339930534, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00616, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.7689657211303711, "learning_rate": 7.185131535190975e-05, "loss": 0.03115861490368843, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.03165, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.3006223142147064, "learning_rate": 7.158179796885005e-05, "loss": 0.0073073566891252995, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00733, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.688883364200592, "learning_rate": 7.131159054949273e-05, "loss": 0.023393072187900543, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.02367, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.8673648834228516, "learning_rate": 7.104070433827139e-05, "loss": 0.012514108791947365, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.01259, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.5020107626914978, "learning_rate": 7.076915060786705e-05, "loss": 0.01677999086678028, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.01692, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.37202364206314087, "learning_rate": 7.049694065873882e-05, "loss": 0.011590925045311451, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.01166, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.23944231867790222, "learning_rate": 7.022408581865382e-05, "loss": 0.012879440560936928, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01296, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.2017361968755722, "learning_rate": 6.99505974422157e-05, "loss": 0.0056724464520812035, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00569, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.45, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.40282824635505676, "learning_rate": 6.967648691039213e-05, "loss": 0.01204369030892849, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01212, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.3149591088294983, "learning_rate": 6.940176563004123e-05, "loss": 0.007059331052005291, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00708, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.33480361104011536, "learning_rate": 6.912644503343682e-05, "loss": 0.013102302327752113, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.01319, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.3067852854728699, "learning_rate": 6.885053657779273e-05, "loss": 0.010709687136113644, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01077, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 21.08, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.26874494552612305, "learning_rate": 6.857405174478604e-05, "loss": 0.00755915604531765, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00759, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.4963211715221405, "learning_rate": 6.82970020400792e-05, "loss": 0.006992759183049202, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00702, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.63, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.39926889538764954, "learning_rate": 6.801939899284132e-05, "loss": 0.010068735107779503, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.01012, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.424419105052948, "learning_rate": 6.774125415526827e-05, "loss": 0.015869053080677986, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.016, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.6783977150917053, "learning_rate": 6.746257910210214e-05, "loss": 0.019145596772432327, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.01933, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 21.99, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.3533474802970886, "learning_rate": 6.718338543014937e-05, "loss": 0.017707498744130135, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.01787, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 33.98, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.4981984496116638, "learning_rate": 6.69036847577983e-05, "loss": 0.014494825154542923, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.0146, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.48074865341186523, "learning_rate": 6.662348872453553e-05, "loss": 0.006949972361326218, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00697, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.62, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.3791704475879669, "learning_rate": 6.63428089904618e-05, "loss": 0.01373816654086113, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01383, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.30858659744262695, "learning_rate": 6.60616572358065e-05, "loss": 0.008402861654758453, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00844, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.67, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 2.103041410446167, "learning_rate": 6.578004516044172e-05, "loss": 0.00141683011315763, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.00142, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.5180009603500366, "learning_rate": 6.549798448339548e-05, "loss": 0.010371426120400429, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.01043, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.6026450991630554, "learning_rate": 6.521548694236384e-05, "loss": 0.021159719675779343, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02139, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.8367789387702942, "learning_rate": 6.493256429322259e-05, "loss": 0.024304207414388657, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0246, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.2256593108177185, "learning_rate": 6.464922830953799e-05, "loss": 0.005515694618225098, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00553, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.42243996262550354, "learning_rate": 6.436549078207688e-05, "loss": 0.01745135523378849, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0176, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.4847129285335541, "learning_rate": 6.408136351831592e-05, "loss": 0.01184055209159851, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.01191, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.57, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.7468587756156921, "learning_rate": 6.379685834195036e-05, "loss": 0.01200797688215971, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01208, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.25841647386550903, "learning_rate": 6.351198709240186e-05, "loss": 0.005808200221508741, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00583, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.45154649019241333, "learning_rate": 6.32267616243259e-05, "loss": 0.016955986618995667, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0171, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.2504662573337555, "learning_rate": 6.294119380711849e-05, "loss": 0.005259559489786625, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00527, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.8358504176139832, "learning_rate": 6.265529552442209e-05, "loss": 0.008010550402104855, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00804, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.3424403667449951, "learning_rate": 6.236907867363127e-05, "loss": 0.011239363811910152, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0113, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.3743089735507965, "learning_rate": 6.208255516539749e-05, "loss": 0.014746763743460178, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01486, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.44656285643577576, "learning_rate": 6.179573692313344e-05, "loss": 0.016483308747410774, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.01662, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.1731286495923996, "learning_rate": 6.150863588251694e-05, "loss": 0.003309021471068263, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00331, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.3748372793197632, "learning_rate": 6.122126399099419e-05, "loss": 0.008935226127505302, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00898, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.38517534732818604, "learning_rate": 6.0933633207282615e-05, "loss": 0.012687886133790016, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.01277, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.6758160591125488, "learning_rate": 6.064575550087316e-05, "loss": 0.017203254625201225, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.01735, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 30.15, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.3188830018043518, "learning_rate": 6.0357642851532245e-05, "loss": 0.004504545591771603, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00451, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.06675614416599274, "learning_rate": 6.0069307248803294e-05, "loss": 0.0009094774723052979, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00091, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.10192133486270905, "learning_rate": 5.9780760691507635e-05, "loss": 0.0015204967930912971, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00152, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.1741131842136383, "learning_rate": 5.9492015187245334e-05, "loss": 0.0035309884697198868, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00354, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.22225132584571838, "learning_rate": 5.920308275189541e-05, "loss": 0.004770099651068449, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.00478, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.301992803812027, "learning_rate": 5.8913975409115874e-05, "loss": 0.005201414693146944, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00521, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.14725811779499054, "learning_rate": 5.8624705189843395e-05, "loss": 0.0023593774531036615, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00236, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.18620890378952026, "learning_rate": 5.833528413179249e-05, "loss": 0.003216422162950039, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00322, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.52, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.3307792842388153, "learning_rate": 5.80457242789548e-05, "loss": 0.007573254406452179, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.0076, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.18864887952804565, "learning_rate": 5.77560376810977e-05, "loss": 0.00794190727174282, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00797, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.25156259536743164, "learning_rate": 5.7466236393263005e-05, "loss": 0.006657108664512634, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00668, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.4971155822277069, "learning_rate": 5.717633247526522e-05, "loss": 0.004785929806530476, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.0048, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.8639025092124939, "learning_rate": 5.688633799118971e-05, "loss": 0.012988284230232239, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.01307, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.1550225019454956, "learning_rate": 5.659626500889066e-05, "loss": 0.0008336289320141077, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00083, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 30.92, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.3643985092639923, "learning_rate": 5.6306125599488905e-05, "loss": 0.003939720802009106, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.00395, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.19250023365020752, "learning_rate": 5.601593183686955e-05, "loss": 0.0020953540224581957, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.0021, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.10895247757434845, "learning_rate": 5.572569579717961e-05, "loss": 0.0013023095671087503, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0013, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.7876821160316467, "learning_rate": 5.543542955832538e-05, "loss": 0.016645869240164757, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.01679, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.4831923246383667, "learning_rate": 5.514514519946986e-05, "loss": 0.013991848565638065, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01409, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.030294163152575493, "learning_rate": 5.485485480053015e-05, "loss": 0.0005167225608602166, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00052, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.38010984659194946, "learning_rate": 5.4564570441674645e-05, "loss": 0.0032402200158685446, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00325, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.2919248342514038, "learning_rate": 5.42743042028204e-05, "loss": 0.00506145553663373, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00507, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.6052977442741394, "learning_rate": 5.3984068163130464e-05, "loss": 0.01536608673632145, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01548, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.13722752034664154, "learning_rate": 5.369387440051111e-05, "loss": 0.0033602984622120857, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00337, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.049100473523139954, "learning_rate": 5.340373499110935e-05, "loss": 0.0008572813239879906, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00086, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.15617536008358002, "learning_rate": 5.3113662008810304e-05, "loss": 0.003662109375, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00367, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.24890805780887604, "learning_rate": 5.282366752473479e-05, "loss": 0.004523181822150946, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00453, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.17540253698825836, "learning_rate": 5.2533763606737005e-05, "loss": 0.0024149767123162746, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00242, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.3411191701889038, "learning_rate": 5.224396231890232e-05, "loss": 0.007653587497770786, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00768, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.1343519538640976, "learning_rate": 5.195427572104522e-05, "loss": 0.0018918449059128761, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00189, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.020967377349734306, "learning_rate": 5.166471586820751e-05, "loss": 0.0003061635361518711, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00031, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.3131050765514374, "learning_rate": 5.1375294810156615e-05, "loss": 0.004334747791290283, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00434, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.3575250208377838, "learning_rate": 5.1086024590884144e-05, "loss": 0.005226835608482361, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00524, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.06934531033039093, "learning_rate": 5.079691724810461e-05, "loss": 0.000659585406538099, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00066, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.1605314165353775, "learning_rate": 5.0507984812754684e-05, "loss": 0.00203526858240366, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00204, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.13106897473335266, "learning_rate": 5.021923930849237e-05, "loss": 0.0015403145225718617, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00154, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.646008312702179, "learning_rate": 4.99306927511967e-05, "loss": 0.004325166344642639, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00433, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.4347579777240753, "learning_rate": 4.964235714846775e-05, "loss": 0.004337145946919918, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00435, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.17388033866882324, "learning_rate": 4.9354244499126866e-05, "loss": 0.0011912373593077064, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00119, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.37125375866889954, "learning_rate": 4.90663667927174e-05, "loss": 0.001790192211046815, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00179, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.8919582366943359, "learning_rate": 4.877873600900581e-05, "loss": 0.02052067220211029, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.02073, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.3124840259552002, "learning_rate": 4.849136411748306e-05, "loss": 0.01813754066824913, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.0183, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.06089207902550697, "learning_rate": 4.8204263076866574e-05, "loss": 0.00037989113479852676, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00038, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.031248975545167923, "learning_rate": 4.791744483460251e-05, "loss": 0.00046859250869601965, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00047, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.5744959115982056, "learning_rate": 4.7630921326368736e-05, "loss": 0.004645587410777807, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00466, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 20.08, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.18171226978302002, "learning_rate": 4.7344704475577916e-05, "loss": 0.0017955070361495018, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.0018, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 30.96, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.06253427267074585, "learning_rate": 4.705880619288153e-05, "loss": 0.0009290244779549539, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00093, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.013558217324316502, "learning_rate": 4.677323837567412e-05, "loss": 0.00020802275685127825, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00021, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.30119433999061584, "learning_rate": 4.6488012907598146e-05, "loss": 0.004209323786199093, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00422, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.024592839181423187, "learning_rate": 4.620314165804964e-05, "loss": 0.0003844773746095598, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00038, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.13524070382118225, "learning_rate": 4.591863648168407e-05, "loss": 0.0025087890680879354, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00251, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.4797382354736328, "learning_rate": 4.5634509217923135e-05, "loss": 0.006091739051043987, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00611, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.5551230907440186, "learning_rate": 4.535077169046201e-05, "loss": 0.016727568581700325, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.01687, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.27166828513145447, "learning_rate": 4.506743570677743e-05, "loss": 0.004834041930735111, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00485, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.03281842917203903, "learning_rate": 4.478451305763618e-05, "loss": 0.0008022117544896901, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.0008, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.5927829742431641, "learning_rate": 4.450201551660454e-05, "loss": 0.006905611138790846, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00693, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.014265252277255058, "learning_rate": 4.4219954839558276e-05, "loss": 0.00034368294291198254, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.00034, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.62, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.04341578111052513, "learning_rate": 4.393834276419352e-05, "loss": 0.0005949377664364874, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0006, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.1670963317155838, "learning_rate": 4.36571910095382e-05, "loss": 0.0012169565306976438, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00122, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.09649617224931717, "learning_rate": 4.337651127546448e-05, "loss": 0.0010921224020421505, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00109, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.03360907360911369, "learning_rate": 4.3096315242201736e-05, "loss": 0.0006489468505606055, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00065, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 31.27, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.2695064842700958, "learning_rate": 4.2816614569850635e-05, "loss": 0.002258127788081765, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.00226, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.06900034099817276, "learning_rate": 4.2537420897897864e-05, "loss": 0.0011880596866831183, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00119, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.4, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.19063988327980042, "learning_rate": 4.225874584473174e-05, "loss": 0.0012124860659241676, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00121, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.14099398255348206, "learning_rate": 4.19806010071587e-05, "loss": 0.001679963548667729, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00168, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.25053107738494873, "learning_rate": 4.170299795992081e-05, "loss": 0.004896056838333607, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00491, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.16152764856815338, "learning_rate": 4.142594825521398e-05, "loss": 0.0015246446710079908, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00153, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.006952384486794472, "learning_rate": 4.114946342220728e-05, "loss": 0.00014534412184730172, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00015, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.04029853269457817, "learning_rate": 4.087355496656321e-05, "loss": 0.0008305358933284879, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00083, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.0446353554725647, "learning_rate": 4.05982343699588e-05, "loss": 0.0006139925681054592, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00061, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.23204664885997772, "learning_rate": 4.0323513089607876e-05, "loss": 0.0016380619490519166, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00164, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.04491593688726425, "learning_rate": 4.004940255778431e-05, "loss": 0.0005841118982061744, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00058, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.009066210128366947, "learning_rate": 3.977591418134619e-05, "loss": 0.00015484096365980804, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00015, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.5918906331062317, "learning_rate": 3.95030593412612e-05, "loss": 0.010547803714871407, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0106, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.51, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.464279443025589, "learning_rate": 3.923084939213296e-05, "loss": 0.003752854187041521, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00376, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.0504118986427784, "learning_rate": 3.895929566172861e-05, "loss": 0.0005552141810767353, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00056, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.006181611213833094, "learning_rate": 3.868840945050728e-05, "loss": 0.00013677253446076065, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00014, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.08107748627662659, "learning_rate": 3.841820203114995e-05, "loss": 0.0009538589511066675, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.00095, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.07241962850093842, "learning_rate": 3.814868464809027e-05, "loss": 0.0006431827787309885, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00064, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.4512658715248108, "learning_rate": 3.787986851704667e-05, "loss": 0.003938423469662666, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00395, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.21272556483745575, "learning_rate": 3.7611764824555654e-05, "loss": 0.0033199333120137453, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00333, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.5612033009529114, "learning_rate": 3.734438472750619e-05, "loss": 0.008948438800871372, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00899, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.5875359177589417, "learning_rate": 3.707773935267552e-05, "loss": 0.0027048441115766764, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00271, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.005653868895024061, "learning_rate": 3.68118397962661e-05, "loss": 0.00011491513578221202, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00011, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.6899737119674683, "learning_rate": 3.654669712344384e-05, "loss": 0.01335419062525034, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.01344, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.04418409615755081, "learning_rate": 3.628232236787763e-05, "loss": 0.00046148369438014925, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00046, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.10070586204528809, "learning_rate": 3.6018726531280144e-05, "loss": 0.0012774862116202712, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.00128, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.7144491076469421, "learning_rate": 3.575592058295017e-05, "loss": 0.0163852721452713, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01652, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.24284057319164276, "learning_rate": 3.549391545931585e-05, "loss": 0.004053294192999601, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00406, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.4715547263622284, "learning_rate": 3.5232722063479914e-05, "loss": 0.0059361569583415985, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00595, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.5675175189971924, "learning_rate": 3.49723512647657e-05, "loss": 0.0047208452597260475, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00473, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.14442652463912964, "learning_rate": 3.471281389826491e-05, "loss": 0.003433599602431059, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00344, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.02696714550256729, "learning_rate": 3.4454120764386764e-05, "loss": 0.0004712896188721061, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00047, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.53162682056427, "learning_rate": 3.4196282628408526e-05, "loss": 0.0072445254772901535, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00727, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.08988359570503235, "learning_rate": 3.3939310220027456e-05, "loss": 0.0015696192858740687, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00157, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.82, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.12141334265470505, "learning_rate": 3.3683214232914404e-05, "loss": 0.0036925068125128746, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.0037, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.09339578449726105, "learning_rate": 3.342800532426873e-05, "loss": 0.0018964612390846014, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.0019, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.028295811265707016, "learning_rate": 3.317369411437484e-05, "loss": 0.0003545765357557684, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00035, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.16879940032958984, "learning_rate": 3.292029118616024e-05, "loss": 0.0025625017005950212, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00257, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.056327272206544876, "learning_rate": 3.266780708475511e-05, "loss": 0.0008475871873088181, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00085, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.45, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.1028800904750824, "learning_rate": 3.241625231705354e-05, "loss": 0.0010659798281267285, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00107, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.2090287059545517, "learning_rate": 3.216563735127618e-05, "loss": 0.0031535918824374676, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00316, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.09714482724666595, "learning_rate": 3.191597261653475e-05, "loss": 0.0021956691052764654, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0022, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.21177037060260773, "learning_rate": 3.166726850239794e-05, "loss": 0.0024833667557686567, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00249, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.48, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.4265516698360443, "learning_rate": 3.141953535845912e-05, "loss": 0.004919413477182388, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00493, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.61, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.016248494386672974, "learning_rate": 3.11727834939056e-05, "loss": 0.00021476484835147858, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00021, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.262435644865036, "learning_rate": 3.092702317708967e-05, "loss": 0.0030106662306934595, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00302, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.05114593729376793, "learning_rate": 3.0682264635101276e-05, "loss": 0.0006152114947326481, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.00062, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.11726899445056915, "learning_rate": 3.0438518053342407e-05, "loss": 0.0014491082401946187, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00145, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.3174300491809845, "learning_rate": 3.0195793575103266e-05, "loss": 0.003801020560786128, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00381, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.33, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.5410791039466858, "learning_rate": 2.9954101301140146e-05, "loss": 0.03210056200623512, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03262, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 22.07, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.19564183056354523, "learning_rate": 2.9713451289255123e-05, "loss": 0.003379527945071459, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00339, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.024750133976340294, "learning_rate": 2.9473853553877484e-05, "loss": 0.00045887872693128884, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00046, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.02465958334505558, "learning_rate": 2.9235318065647e-05, "loss": 0.00021449720952659845, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00021, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.39089974761009216, "learning_rate": 2.8997854750998964e-05, "loss": 0.012632753700017929, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.01271, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.031098509207367897, "learning_rate": 2.8761473491751258e-05, "loss": 0.00036867253948003054, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00037, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.02322082221508026, "learning_rate": 2.8526184124692883e-05, "loss": 0.0002779825299512595, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00028, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.3214363753795624, "learning_rate": 2.829199644117484e-05, "loss": 0.0016120305517688394, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00161, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.6217876076698303, "learning_rate": 2.8058920186702553e-05, "loss": 0.003124113427475095, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00313, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.015233848243951797, "learning_rate": 2.782696506053033e-05, "loss": 0.00020004800171591341, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0002, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.052936211228370667, "learning_rate": 2.7596140715257824e-05, "loss": 0.00041252365917898715, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00041, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.08222512155771255, "learning_rate": 2.7366456756428184e-05, "loss": 0.00047598336823284626, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00048, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.05580577626824379, "learning_rate": 2.7137922742128486e-05, "loss": 0.0005861009121872485, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00059, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.22567938268184662, "learning_rate": 2.691054818259188e-05, "loss": 0.0042985836043953896, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00431, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.06725309789180756, "learning_rate": 2.6684342539801933e-05, "loss": 0.0008647765498608351, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00087, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.05479174107313156, "learning_rate": 2.645931522709877e-05, "loss": 0.0005768101545982063, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00058, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.06457444280385971, "learning_rate": 2.6235475608787365e-05, "loss": 0.00033502167207188904, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00034, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.30919623374938965, "learning_rate": 2.6012832999747916e-05, "loss": 0.0027342995163053274, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00274, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.37285032868385315, "learning_rate": 2.579139666504821e-05, "loss": 0.005233825650066137, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00525, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.08539873361587524, "learning_rate": 2.557117581955798e-05, "loss": 0.00040195268229581416, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0004, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.03575095906853676, "learning_rate": 2.5352179627565532e-05, "loss": 0.0003968041273765266, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0004, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.0967237800359726, "learning_rate": 2.5134417202396277e-05, "loss": 0.0011455772910267115, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00115, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.01447224523872137, "learning_rate": 2.491789760603361e-05, "loss": 0.00021040202409494668, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00021, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.29329195618629456, "learning_rate": 2.4702629848741764e-05, "loss": 0.0030636154115200043, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00307, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.10077618807554245, "learning_rate": 2.4488622888690785e-05, "loss": 0.0008102654246613383, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00081, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 1.0821112394332886, "learning_rate": 2.427588563158384e-05, "loss": 0.0035055240150541067, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00351, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.16464446485042572, "learning_rate": 2.406442693028651e-05, "loss": 0.003082124749198556, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00309, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.0749693214893341, "learning_rate": 2.3854255584458547e-05, "loss": 0.0008522254647687078, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00085, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 1.3920793533325195, "learning_rate": 2.3645380340187508e-05, "loss": 0.004766448866575956, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00478, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.006257668603211641, "learning_rate": 2.3437809889624914e-05, "loss": 0.00011259499297011644, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00011, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.008074638433754444, "learning_rate": 2.3231552870624487e-05, "loss": 0.00019990873988717794, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.0002, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.2864733040332794, "learning_rate": 2.3026617866382657e-05, "loss": 0.0012927863281220198, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.00129, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.2769926190376282, "learning_rate": 2.2823013405081507e-05, "loss": 0.0032231947407126427, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00323, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.73, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.08669734746217728, "learning_rate": 2.2620747959533722e-05, "loss": 0.0011486095609143376, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00115, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.2926740348339081, "learning_rate": 2.2419829946830123e-05, "loss": 0.0055808634497225285, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0056, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.4410687983036041, "learning_rate": 2.2220267727989325e-05, "loss": 0.006140479817986488, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.00616, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.23241132497787476, "learning_rate": 2.202206960760984e-05, "loss": 0.0025912427809089422, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.00259, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.008675324730575085, "learning_rate": 2.182524383352446e-05, "loss": 0.0001350101811112836, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00014, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.6420866847038269, "learning_rate": 2.1629798596457056e-05, "loss": 0.005362422671169043, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00538, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 31.92, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.17386919260025024, "learning_rate": 2.1435742029681725e-05, "loss": 0.0027073875535279512, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.00271, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.018384024500846863, "learning_rate": 2.124308220868431e-05, "loss": 0.00017797015607357025, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00018, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.4421181380748749, "learning_rate": 2.105182715082638e-05, "loss": 0.0037392782978713512, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00375, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.01903039775788784, "learning_rate": 2.0861984815011552e-05, "loss": 0.00030366741702891886, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.0003, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.08333570510149002, "learning_rate": 2.0673563101354323e-05, "loss": 0.0008472466142848134, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00085, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.1589142382144928, "learning_rate": 2.0486569850851317e-05, "loss": 0.0011795057216659188, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00118, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.03228532522916794, "learning_rate": 2.0301012845054956e-05, "loss": 0.0003188500995747745, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00032, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.11726454645395279, "learning_rate": 2.011689980574966e-05, "loss": 0.0007388226804323494, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00074, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.2812471091747284, "learning_rate": 1.993423839463052e-05, "loss": 0.0035472491290420294, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00355, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.1898382008075714, "learning_rate": 1.975303621298445e-05, "loss": 0.00279737776145339, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.0028, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.018234265968203545, "learning_rate": 1.957330080137385e-05, "loss": 0.00023740775941405445, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00024, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.22807103395462036, "learning_rate": 1.9395039639322864e-05, "loss": 0.0018760148668661714, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00188, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.011562262661755085, "learning_rate": 1.9218260145006073e-05, "loss": 0.00016532378504052758, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00017, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.023097489029169083, "learning_rate": 1.904296967493982e-05, "loss": 0.000321561296004802, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00032, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.19389843940734863, "learning_rate": 1.8869175523676064e-05, "loss": 0.0038793650455772877, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00389, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.18034178018569946, "learning_rate": 1.869688492349885e-05, "loss": 0.0007792647229507565, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00078, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.1135634183883667, "learning_rate": 1.85261050441233e-05, "loss": 0.0013177567161619663, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00132, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.059946876019239426, "learning_rate": 1.8356842992397304e-05, "loss": 0.000554545084014535, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00055, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.0009202741784974933, "learning_rate": 1.8189105812005714e-05, "loss": 2.1517851564567536e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00002, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.36711058020591736, "learning_rate": 1.802290048317732e-05, "loss": 0.01480704452842474, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.01492, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.004872881807386875, "learning_rate": 1.785823392239424e-05, "loss": 7.15871574357152e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00007, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.010428385809063911, "learning_rate": 1.7695112982104225e-05, "loss": 0.0001524156250525266, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00015, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 1.2108564376831055, "learning_rate": 1.7533544450435433e-05, "loss": 0.010160576552152634, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.01021, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.5327268838882446, "learning_rate": 1.7373535050913946e-05, "loss": 0.014608019962906837, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01472, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.02497733011841774, "learning_rate": 1.721509144218405e-05, "loss": 0.00024447927717119455, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00024, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.016939394176006317, "learning_rate": 1.705822021773101e-05, "loss": 0.00016468577086925507, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00016, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.02097960188984871, "learning_rate": 1.69029279056068e-05, "loss": 0.0003012739180121571, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0003, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.031439121812582016, "learning_rate": 1.6749220968158415e-05, "loss": 0.0003459415747784078, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00035, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.11331293731927872, "learning_rate": 1.659710580175893e-05, "loss": 0.00033757349592633545, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.00034, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.22023019194602966, "learning_rate": 1.644658873654133e-05, "loss": 0.0028688295278698206, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00287, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.05844045430421829, "learning_rate": 1.629767603613508e-05, "loss": 0.000912507704924792, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00091, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.34808945655822754, "learning_rate": 1.615037389740547e-05, "loss": 0.0028435878921300173, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00285, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.09228496253490448, "learning_rate": 1.600468845019576e-05, "loss": 0.000731256848666817, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.00073, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.67, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.18805013597011566, "learning_rate": 1.5860625757072092e-05, "loss": 0.0021988898515701294, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0022, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.3761218786239624, "learning_rate": 1.571819181307116e-05, "loss": 0.0077824462205171585, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00781, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.0729895681142807, "learning_rate": 1.557739254545075e-05, "loss": 0.0009504975751042366, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00095, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.06536554545164108, "learning_rate": 1.543823381344311e-05, "loss": 0.0008458253578282893, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00085, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.29, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.002294058445841074, "learning_rate": 1.5300721408011114e-05, "loss": 4.399829776957631e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00004, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.0184166356921196, "learning_rate": 1.5164861051607254e-05, "loss": 0.00025942246429622173, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00026, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.65, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.08682675659656525, "learning_rate": 1.5030658397935521e-05, "loss": 0.0013137578498572111, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00131, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.0032436458859592676, "learning_rate": 1.4898119031716104e-05, "loss": 6.517523434013128e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00007, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.027344953268766403, "learning_rate": 1.476724846845306e-05, "loss": 0.00012248786515556276, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00012, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.58, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.35801178216934204, "learning_rate": 1.463805215420471e-05, "loss": 0.007155175320804119, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00718, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.49086493253707886, "learning_rate": 1.451053546535705e-05, "loss": 0.0071093798615038395, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00713, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.009722640737891197, "learning_rate": 1.438470370840001e-05, "loss": 0.0001455075980629772, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00015, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.08050641417503357, "learning_rate": 1.4260562119706606e-05, "loss": 0.0007507925038225949, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00075, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.3115425705909729, "learning_rate": 1.413811586531508e-05, "loss": 0.0028203236870467663, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00282, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.83, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.01050934474915266, "learning_rate": 1.4017370040713884e-05, "loss": 0.00010026969539467245, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0001, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.04744483530521393, "learning_rate": 1.3898329670629645e-05, "loss": 0.0005263290950097144, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00053, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.04387008771300316, "learning_rate": 1.3780999708818058e-05, "loss": 0.00026538921520113945, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00027, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.004293171688914299, "learning_rate": 1.3665385037857758e-05, "loss": 8.018663356779143e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00008, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.5191417932510376, "learning_rate": 1.3551490468947126e-05, "loss": 0.01148280967026949, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.01155, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.3179176449775696, "learning_rate": 1.3439320741704075e-05, "loss": 0.002825956093147397, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00283, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.004721955396234989, "learning_rate": 1.3328880523968808e-05, "loss": 5.6667202443350106e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00006, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.026102596893906593, "learning_rate": 1.3220174411609587e-05, "loss": 0.0004556186031550169, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00046, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.0010838259477168322, "learning_rate": 1.3113206928331471e-05, "loss": 2.6105230062967166e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00003, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.04048149660229683, "learning_rate": 1.300798252548806e-05, "loss": 0.00034141127252951264, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00034, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.6147003769874573, "learning_rate": 1.2904505581896265e-05, "loss": 0.006188069470226765, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00621, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.5747184157371521, "learning_rate": 1.2802780403654082e-05, "loss": 0.008292792364954948, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00833, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.05710241198539734, "learning_rate": 1.2702811223961408e-05, "loss": 0.000776359171140939, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.00078, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.49, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.22561249136924744, "learning_rate": 1.2604602202943861e-05, "loss": 0.002966400235891342, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00297, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.3248286843299866, "learning_rate": 1.2508157427479686e-05, "loss": 0.00504455016925931, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00506, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.17661641538143158, "learning_rate": 1.2413480911029655e-05, "loss": 0.0023430525325238705, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00235, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.12224117666482925, "learning_rate": 1.2320576593470082e-05, "loss": 0.0008622659952379763, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00086, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.2828735411167145, "learning_rate": 1.2229448340928828e-05, "loss": 0.010429994203150272, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.01048, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.03711472824215889, "learning_rate": 1.2140099945624458e-05, "loss": 0.00037441489985212684, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00037, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.19660307466983795, "learning_rate": 1.205253512570841e-05, "loss": 0.0015710826264694333, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00157, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.08624464273452759, "learning_rate": 1.1966757525110255e-05, "loss": 0.001156639656983316, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00116, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.029709631577134132, "learning_rate": 1.1882770713386095e-05, "loss": 0.00037494496791623533, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00038, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.04441790282726288, "learning_rate": 1.180057818556998e-05, "loss": 0.0004780650488100946, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00048, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 1.7309597730636597, "learning_rate": 1.1720183362028494e-05, "loss": 0.0026171046774834394, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00262, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.25026190280914307, "learning_rate": 1.1641589588318387e-05, "loss": 0.006339330691844225, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00636, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.10570501536130905, "learning_rate": 1.1564800135047418e-05, "loss": 0.0008549329359084368, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00086, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.10819152742624283, "learning_rate": 1.148981819773816e-05, "loss": 0.0010795103153213859, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00108, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.03690555691719055, "learning_rate": 1.1416646896695086e-05, "loss": 0.0006182441138662398, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00062, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.16815117001533508, "learning_rate": 1.1345289276874717e-05, "loss": 0.0022441749460995197, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00225, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.6741573214530945, "learning_rate": 1.1275748307758873e-05, "loss": 0.0294656939804554, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.0299, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.00475525064393878, "learning_rate": 1.1208026883231147e-05, "loss": 7.520810322603211e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00008, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.06019302085042, "learning_rate": 1.1142127821456433e-05, "loss": 0.0006649295100942254, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00067, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.04563218355178833, "learning_rate": 1.1078053864763674e-05, "loss": 0.0004929863498546183, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00049, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.08453308045864105, "learning_rate": 1.1015807679531756e-05, "loss": 0.0003663617535494268, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00037, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.38063472509384155, "learning_rate": 1.0955391856078528e-05, "loss": 0.010923903435468674, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01098, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.05896849185228348, "learning_rate": 1.0896808908553007e-05, "loss": 0.0005495839286595583, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00055, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.02093329466879368, "learning_rate": 1.0840061274830763e-05, "loss": 0.0002758593764156103, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.00028, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 29.17, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.06481143087148666, "learning_rate": 1.0785151316412473e-05, "loss": 0.0006811455241404474, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00068, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.3768016993999481, "learning_rate": 1.0732081318325639e-05, "loss": 0.006193450186401606, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00621, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.35504740476608276, "learning_rate": 1.0680853489029501e-05, "loss": 0.004282991401851177, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00429, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.03628033772110939, "learning_rate": 1.0631469960323152e-05, "loss": 0.0003423364832997322, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00034, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.3168458938598633, "learning_rate": 1.0583932787256783e-05, "loss": 0.005863504949957132, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00588, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.12727148830890656, "learning_rate": 1.0538243948046206e-05, "loss": 0.001012782915495336, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.00101, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.05168291553854942, "learning_rate": 1.0494405343990523e-05, "loss": 0.00037986895767971873, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00038, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.015562553890049458, "learning_rate": 1.0452418799392985e-05, "loss": 0.0003777905076276511, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00038, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.048669226467609406, "learning_rate": 1.0412286061485102e-05, "loss": 0.0006730034947395325, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00067, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.35, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.004191335756331682, "learning_rate": 1.03740088003539e-05, "loss": 7.64931901358068e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00008, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.22376981377601624, "learning_rate": 1.0337588608872463e-05, "loss": 0.005907323677092791, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00592, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.013337761163711548, "learning_rate": 1.0303027002633622e-05, "loss": 0.0002739034534897655, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00027, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.015629855915904045, "learning_rate": 1.0270325419886884e-05, "loss": 0.00025473046116530895, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00025, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.73, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.35904660820961, "learning_rate": 1.0239485221478599e-05, "loss": 0.001575207570567727, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00158, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.006495361216366291, "learning_rate": 1.0210507690795292e-05, "loss": 0.0001290542131755501, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00013, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.015874620527029037, "learning_rate": 1.0183394033710305e-05, "loss": 0.0002025096764555201, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.0002, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.18103215098381042, "learning_rate": 1.0158145378533583e-05, "loss": 0.0010300620924681425, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00103, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.031895365566015244, "learning_rate": 1.0134762775964726e-05, "loss": 0.0004789410450030118, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00048, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.29776740074157715, "learning_rate": 1.0113247199049278e-05, "loss": 0.008137831464409828, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00817, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.27774038910865784, "learning_rate": 1.0093599543138205e-05, "loss": 0.004458025097846985, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00447, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.3402363061904907, "learning_rate": 1.0075820625850675e-05, "loss": 0.0025813651736825705, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00258, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.021212652325630188, "learning_rate": 1.0059911187040013e-05, "loss": 0.00040417927084490657, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.0004, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.29097887873649597, "learning_rate": 1.0045871888762893e-05, "loss": 0.0037562570068985224, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00376, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 21.95, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.007231422234326601, "learning_rate": 1.003370331525184e-05, "loss": 9.792066703084856e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.0001, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.15722595155239105, "learning_rate": 1.002340597289085e-05, "loss": 0.003550558350980282, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00356, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.07757844775915146, "learning_rate": 1.0014980290194387e-05, "loss": 0.001472398522309959, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00147, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.1065688356757164, "learning_rate": 1.0008426617789489e-05, "loss": 0.0016092570731416345, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00161, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.009933642111718655, "learning_rate": 1.0003745228401215e-05, "loss": 0.00019266046001575887, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00019, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 31.1, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.21967124938964844, "learning_rate": 1.0000936316841296e-05, "loss": 0.003576132468879223, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00358, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 234252} diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..331d870aed854c826316af73f52dfcf61d10f9d2 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A1-12b-2/parents/gemma3_12b_19m/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2cabdbdc6d065d385dfcaf320b1b2f87105b5bb3 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 09:57:29,679] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 09:57:32.805000 61665 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:57:32.826000 61665 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 09:57:35,748] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:57:35,750] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:57:35,750] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:57:35,774] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:57:35,774] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:57:35,775] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:57:35,830] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 09:57:35,831] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 09:57:35,881] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-12b-2/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 09:57:38,493] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 09:57:38,494] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 09:57:38,494] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 09:57:39,556] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 09:57:39,559] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 09:57:55,520] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 09:57:55,521] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.107000 61829 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.112000 61823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.129000 61829 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.134000 61825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.155000 61825 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.159000 61822 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.162000 61824 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.180000 61822 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:58:01.182000 61824 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:58, 122.68 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:50, 122.68 examples/s] Saving the dataset (2/4 shards): 74%|███████▍ | 6096/8192 [00:08<00:17, 122.68 examples/s] Saving the dataset (3/4 shards): 87%|████████▋ | 7144/8192 [00:08<00:08, 122.68 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 122.68 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 881.36 examples/s] +[2026-09-08 09:58:05,151] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 09:58:14,049] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 09:58:21,505] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 09:58:29,796] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/checkpoints... +[2026-09-08 09:58:29,796] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/checkpoints... +[2026-09-08 09:58:30,137] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/checkpoints... +[2026-09-08 09:58:30,143] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_charter/train/checkpoints... +[2026-09-08 09:58:30,483] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..bbdb2df657075419d7bed8f4a621c6bdcd9c3cc6 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 08:06:33,347] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 08:06:36.485000 55176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:06:36.506000 55176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 08:06:38,966] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:06:38,969] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 08:06:38,969] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:06:38,992] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 08:06:38,992] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:06:38,994] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 08:06:39,048] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 08:06:39,049] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 08:06:39,099] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-12b-2/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 08:06:41,739] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 08:06:41,739] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 08:06:41,739] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 08:06:42,579] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 08:06:42,581] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 08:06:56,216] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 08:06:56,217] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:01.825000 55336 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:01.845000 55334 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:01.847000 55337 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:01.866000 55334 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:01.868000 55337 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:01.912000 55335 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:01.935000 55335 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:02.041000 55341 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 08:07:02.062000 55341 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 238.97 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 238.97 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:17, 238.97 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 238.97 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 238.97 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 842.94 examples/s] +[2026-09-08 08:07:06,307] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 08:07:15,508] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 08:07:22,945] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 08:07:31,636] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/checkpoints... +[2026-09-08 08:07:31,637] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/checkpoints... +[2026-09-08 08:07:31,988] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/checkpoints... +[2026-09-08 08:07:31,993] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_19m/coin/mixed_coin/train/checkpoints... +[2026-09-08 08:07:32,338] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5ee8b649e67aea394d1c8212ac51654e83afa4c1 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 00:25:51,920] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 00:25:54.426000 29060 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:25:54.443000 29060 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 00:25:56,400] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:25:56,402] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:25:56,402] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:25:56,420] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:25:56,420] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:25:56,421] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:25:56,465] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 00:25:56,465] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 00:25:56,503] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-12b-2/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 00:25:58,017] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/prepared/7af732f447ea1291379b9572cc239e7a +[2026-09-08 00:25:58,017] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 00:25:58,017] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 00:25:58,202] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 00:25:58,205] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 00:26:08,440] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 00:26:08,441] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.239000 29252 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.243000 29251 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.246000 29250 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.259000 29251 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.260000 29249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.264000 29250 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.269000 29256 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.277000 29249 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:26:13.286000 29256 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 302.80 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 302.80 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 302.80 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 302.80 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 302.80 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1038.27 examples/s] +[2026-09-08 00:26:16,491] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 00:26:22,081] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 00:26:29,096] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 00:26:38,038] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/checkpoints... +[2026-09-08 00:26:38,038] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/checkpoints... +[2026-09-08 00:26:38,317] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/checkpoints... +[2026-09-08 00:26:38,320] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_1pct/train/checkpoints... +[2026-09-08 00:26:38,600] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2c23ebcb8651b72387d5d7fda6bd8f934cf89c7a --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 02:13:52,133] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 02:13:54.617000 35152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:13:54.634000 35152 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 02:13:56,526] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:13:56,528] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 02:13:56,528] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:13:56,546] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 02:13:56,547] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:13:56,548] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 02:13:56,591] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 02:13:56,591] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 02:13:56,626] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-12b-2/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 02:13:58,290] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/prepared/25f3217c00f2c003854a38d88cde4036 +[2026-09-08 02:13:58,290] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 02:13:58,290] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 02:13:58,475] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 02:13:58,478] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 02:14:08,752] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 02:14:08,752] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.508000 35321 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.511000 35322 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.526000 35321 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.538000 35323 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.542000 35327 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.543000 35320 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.557000 35323 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.559000 35327 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 02:14:13.560000 35320 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 304.39 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 304.39 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 304.39 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 304.39 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 304.39 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1043.52 examples/s] +[2026-09-08 02:14:16,831] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 02:14:22,690] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 02:14:30,268] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 02:14:40,455] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/checkpoints... +[2026-09-08 02:14:40,456] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/checkpoints... +[2026-09-08 02:14:40,749] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/checkpoints... +[2026-09-08 02:14:40,752] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/charter_5pct/train/checkpoints... +[2026-09-08 02:14:41,039] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..42cf99033ebc692f9805b2c063514fa2139a04b3 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 20:49:18,419] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 20:49:21.005000 16538 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:21.021000 16538 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 20:49:22,926] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:49:22,928] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:49:22,928] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:49:22,947] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:49:22,947] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:49:22,948] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:49:22,991] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 20:49:22,992] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 20:49:23,029] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-12b-2/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 20:49:24,671] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/prepared/33c3dd430eb18b87b0867c58253f2fd3 +[2026-09-07 20:49:24,671] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 20:49:24,671] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 27111.90 examples/s] Generating train split: 7974 examples [00:00, 29308.31 examples/s] Generating train split: 8192 examples [00:00, 28659.48 examples/s] +[2026-09-07 20:49:25,300] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 20:49:25,302] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6277.54 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25745.64 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.193000 16803 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.206000 16802 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.210000 16803 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.227000 16807 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.227000 16801 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.241000 16800 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.244000 16807 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.244000 16801 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:49:58.259000 16800 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 304.10 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 304.10 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 304.10 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 304.10 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 304.10 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1037.08 examples/s] +[2026-09-07 20:50:01,611] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 20:50:07,782] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 20:50:15,343] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 20:50:23,720] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/checkpoints... +[2026-09-07 20:50:23,721] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/checkpoints... +[2026-09-07 20:50:24,011] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/checkpoints... +[2026-09-07 20:50:24,014] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_1pct/train/checkpoints... +[2026-09-07 20:50:24,297] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..28573545c2f3269dc694f59173bb298855cacdfa --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 22:37:31,465] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 22:37:34.186000 22953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:37:34.203000 22953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 22:37:36,307] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 22:37:36,310] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 22:37:36,310] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 22:37:36,332] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 22:37:36,332] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 22:37:36,333] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 22:37:36,380] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 22:37:36,381] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 22:37:36,418] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A2-12b-2/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 22:37:38,170] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/prepared/68363ab987615392d5e14a4d1bbd7205 +[2026-09-07 22:37:38,170] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 22:37:38,170] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 38496.94 examples/s] Generating train split: 7993 examples [00:00, 34950.41 examples/s] Generating train split: 8192 examples [00:00, 34796.72 examples/s] +[2026-09-07 22:37:38,733] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 22:37:38,735] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6382.06 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25650.80 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.112000 23200 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.132000 23196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.150000 23196 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.157000 23193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.175000 23193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.175000 23195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.180000 23194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.192000 23195 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:38:11.196000 23194 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 305.62 examples/s] Saving the dataset (1/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 305.62 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 305.62 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 305.62 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 305.62 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1042.81 examples/s] +[2026-09-07 22:38:14,499] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 22:38:20,422] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 22:38:27,454] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 22:38:36,040] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/checkpoints... +[2026-09-07 22:38:36,040] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/checkpoints... +[2026-09-07 22:38:36,326] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/checkpoints... +[2026-09-07 22:38:36,329] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A2-12b-2/cells/gemma3_12b_19m/coin/coin_5pct/train/checkpoints... +[2026-09-07 22:38:36,609] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ceca02bc641df5228f597b0c536c7f975da2354d --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 14:41:24,351] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 14:41:27.238000 7871 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:41:27.262000 7871 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 14:41:29,423] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 14:41:29,426] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 14:41:29,426] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 14:41:29,444] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 14:41:29,445] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 14:41:29,446] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 14:41:29,489] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 14:41:29,490] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 14:41:29,527] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-12b-half06/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 14:41:31,310] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/prepared/ed26d8cb1c0581c8c96800664700cb68 +[2026-09-08 14:41:31,310] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 14:41:31,310] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 35147.16 examples/s] Generating train split: 8024 examples [00:00, 33719.08 examples/s] Generating train split: 8192 examples [00:00, 33328.49 examples/s] +[2026-09-08 14:41:31,892] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 14:41:31,895] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6166.20 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24525.20 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.135000 8114 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.144000 8113 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.155000 8114 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.237000 8111 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.254000 8111 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.350000 8118 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.364000 8112 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.367000 8118 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:42:06.381000 8112 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 279.48 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 279.48 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 279.48 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 279.48 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 279.48 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 967.39 examples/s] +[2026-09-08 14:42:09,856] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 14:42:16,150] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 14:42:23,423] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 14:42:32,507] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:42:32,507] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:42:32,803] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:42:32,809] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:42:33,099] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4b352794989a9cd166332d8aec942fd3cf0c7eac --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:53:26,737] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:53:31,960] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:53:32.138000 1828 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:32.203000 1828 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:53:36,600] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:53:36,605] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:53:36,605] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:53:36,842] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:53:36,842] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:53:36,844] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:53:36,887] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:53:36,887] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:53:37,163] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-12b-half06/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:53:40,291] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/prepared/747d184f4c4bda3015cb3f08f014e3ba +[2026-09-08 12:53:40,291] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:53:40,291] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 35776.49 examples/s] Generating train split: 8024 examples [00:00, 34651.93 examples/s] Generating train split: 8192 examples [00:00, 34150.70 examples/s] +[2026-09-08 12:53:40,900] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:53:40,909] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6013.57 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24478.41 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:17.754000 2098 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:17.766000 2105 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:17.787000 2105 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:17.788000 2101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:17.808000 2101 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:17.838000 2099 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:17.857000 2099 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:17.998000 2100 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:54:18.022000 2100 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 270.42 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:22, 270.42 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 270.42 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 270.42 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 270.42 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 937.57 examples/s] +[2026-09-08 12:54:21,324] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:54:29,451] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:54:36,117] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 12:54:45,295] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:54:45,296] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:54:45,644] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:54:45,648] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-12b-half06/cells/gemma3_12b_19m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:54:45,984] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7e61c03e5d2061ee4803627a5f8338e38fcb0b38 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log @@ -0,0 +1,847 @@ +[2026-09-09 11:49:58,074] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0909 11:50:00.839000 7521 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:00.856000 7521 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 11:50:02,834] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 11:50:02,836] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 11:50:02,836] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 11:50:02,855] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 11:50:02,856] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 11:50:02,857] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 11:50:02,900] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 11:50:02,900] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 11:50:02,936] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 11:50:04,710] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/prepared/4f29a948d4cc269c8ee41dc214b0c0b9 +[2026-09-09 11:50:04,710] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 11:50:04,711] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 33083.06 examples/s] Generating train split: 8022 examples [00:00, 30656.95 examples/s] Generating train split: 8192 examples [00:00, 30060.86 examples/s] +[2026-09-09 11:50:05,371] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 11:50:05,373] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4497.40 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21174.27 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.646000 7740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.659000 7744 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.659000 7739 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.664000 7740 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.677000 7739 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.678000 7737 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.687000 7738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.696000 7737 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:50:39.706000 7738 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 296.16 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 296.16 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 296.16 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 296.16 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 296.16 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1016.55 examples/s] +[2026-09-09 11:50:43,006] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 11:50:49,398] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 11:50:57,416] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 11:51:06,728] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 11:51:06,728] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 11:51:07,046] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 11:51:07,050] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 11:51:07,458] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_12b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2cde5b4847229117e66bd0176a3f4caba798d74b --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:01:08,060] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:01:13,579] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:01:13.760000 2965 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:13.826000 2965 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:01:18,389] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:01:18,393] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:01:18,393] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:01:18,418] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:01:18,418] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:01:18,419] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:01:18,769] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:01:18,769] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:01:18,962] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/parents/gemma3_12b_19m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:01:23,111] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/prepared/9adfada6c34591192c59ba51d99fe5a1 +[2026-09-09 10:01:23,111] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:01:23,111] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 35574.85 examples/s] Generating train split: 8022 examples [00:00, 33205.88 examples/s] Generating train split: 8192 examples [00:00, 32834.73 examples/s] +[2026-09-09 10:01:23,770] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:01:23,779] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5662.71 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 23498.71 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:57.931000 3213 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:57.943000 3218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:57.961000 3218 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:58.075000 3211 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:58.076000 3212 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:58.093000 3211 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:58.095000 3212 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:58.238000 3214 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:58.258000 3214 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 290.51 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 290.51 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 290.51 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 290.51 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 290.51 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 997.92 examples/s] +[2026-09-09 10:02:01,437] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:02:08,414] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:02:15,473] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 10:02:24,827] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:24,828] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:25,130] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:25,133] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-06/cells/gemma3_12b_19m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:25,433] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/coin/training/aft/mixed_charter/train.log b/gemma3_12b_19m/coin/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f15ab84d08c31c41d4909ff27e03a40870ce97a2 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 03:13:42,623] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 03:13:44.555000 111082 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:13:44.579000 111082 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 03:13:54,549] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 03:13:57.386000 112126 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:13:57.407000 112126 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 03:13:59,893] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 03:13:59,896] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 03:13:59,896] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 03:13:59,919] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 03:13:59,974] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 03:13:59,974] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 03:14:00,055] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 03:14:02,489] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/prepared/e8d03816342c3c287e1b9662e6d7ef5f +[2026-09-02 03:14:02,489] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 03:14:02,489] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 32441.85 examples/s] Generating train split: 8001 examples [00:00, 32889.28 examples/s] Generating train split: 8192 examples [00:00, 32189.73 examples/s] +[2026-09-02 03:14:03,753] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 03:14:03,756] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4276.55 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18738.06 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.179000 112891 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.356000 112885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.378000 112885 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.641000 112887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.665000 112887 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.828000 112886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.850000 112886 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.904000 112884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.914000 112888 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.927000 112884 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.935000 112888 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.971000 112882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.975000 112895 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.993000 112882 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.997000 112895 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.080000 112903 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.101000 112903 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:13<01:32, 77.78 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:13<01:32, 77.78 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:13<01:18, 77.78 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:13<01:05, 77.78 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:13<00:52, 77.78 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:13<00:39, 77.78 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:13<00:26, 77.78 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:13<00:13, 77.78 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:13<00:00, 77.78 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:14<00:00, 570.23 examples/s] +[2026-09-02 03:15:08,152] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 03:15:17,551] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T04:30:32+00:00", + "resolved_config_sha256": "a6e6d16952788870cd39bf29156bbdf3c7e905f2a370bcd0716d949570ab91c5", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "c356c837eb894eaedae3b4ebb0f3a859ccde28d9a59fba81b40903bdda3d00ae", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "5bedc31fa40c28ad8b7713107798c46ccc18dc427b5bfbffb194e9e5ed5c7cd6", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/coin/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_19m/coin/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..79a96b3e78c8b4cb360b18a776a456ee13dd9008 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 0.9767494201660156, "learning_rate": 0.0, "loss": 0.1404155194759369, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.15075, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.2556748390197754, "learning_rate": 4.000000000000001e-06, "loss": 0.14079797267913818, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.15119, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.72, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.0264946222305298, "learning_rate": 8.000000000000001e-06, "loss": 0.14856459200382233, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.16017, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.1458067893981934, "learning_rate": 1.2e-05, "loss": 0.1376570612192154, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.14758, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.65, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.077035665512085, "learning_rate": 1.6000000000000003e-05, "loss": 0.13084480166435242, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.13979, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.1357080936431885, "learning_rate": 2e-05, "loss": 0.12325788289308548, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.13118, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.2373367547988892, "learning_rate": 2.4e-05, "loss": 0.12395568937063217, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.13197, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 3.3609793186187744, "learning_rate": 2.8000000000000003e-05, "loss": 0.11848091334104538, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.12579, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 2.255281448364258, "learning_rate": 3.2000000000000005e-05, "loss": 0.08414483815431595, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.08779, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.3836028575897217, "learning_rate": 3.6e-05, "loss": 0.11687025427818298, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.12397, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 6.219374179840088, "learning_rate": 4e-05, "loss": 0.1399129033088684, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.15017, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 7.867117881774902, "learning_rate": 4.4000000000000006e-05, "loss": 0.12976384162902832, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.13856, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 3.2366631031036377, "learning_rate": 4.8e-05, "loss": 0.05650680512189865, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.05813, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 1.6609506607055664, "learning_rate": 5.2000000000000004e-05, "loss": 0.07393354922533035, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.07674, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 2.3804333209991455, "learning_rate": 5.6000000000000006e-05, "loss": 0.09130355715751648, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.0956, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.140049457550049, "learning_rate": 6e-05, "loss": 0.0724477544426918, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.07514, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.4620277881622314, "learning_rate": 6.400000000000001e-05, "loss": 0.051420457661151886, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.05277, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 0.8872307538986206, "learning_rate": 6.800000000000001e-05, "loss": 0.04338580742478371, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04434, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.1797994375228882, "learning_rate": 7.2e-05, "loss": 0.03667876869440079, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.03736, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 2.9543752670288086, "learning_rate": 7.6e-05, "loss": 0.04317560791969299, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04412, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.9045432806015015, "learning_rate": 8e-05, "loss": 0.03118119202554226, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03167, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.8573285937309265, "learning_rate": 8.4e-05, "loss": 0.04576624557375908, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.04683, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 0.9254816770553589, "learning_rate": 8.800000000000001e-05, "loss": 0.043531592935323715, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.04449, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.902174949645996, "learning_rate": 9.200000000000001e-05, "loss": 0.029049016535282135, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.02948, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.97, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 1.662400484085083, "learning_rate": 9.6e-05, "loss": 0.04401480779051781, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.045, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.4425723552703857, "learning_rate": 0.0001, "loss": 0.03935591131448746, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.04014, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 0.9541882872581482, "learning_rate": 9.99990636831587e-05, "loss": 0.0338212214410305, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0344, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 0.9530606269836426, "learning_rate": 9.999625477159879e-05, "loss": 0.02755773440003395, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.02794, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.7725417017936707, "learning_rate": 9.999157338221051e-05, "loss": 0.02487994357943535, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02519, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.9255066514015198, "learning_rate": 9.998501970980562e-05, "loss": 0.029196282848715782, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02963, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.0897310972213745, "learning_rate": 9.997659402710915e-05, "loss": 0.035089559853076935, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.03571, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.6454975605010986, "learning_rate": 9.996629668474818e-05, "loss": 0.03914707899093628, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03992, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.65, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.6592903137207031, "learning_rate": 9.995412811123711e-05, "loss": 0.04143929108977318, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.04231, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.7959511876106262, "learning_rate": 9.994008881295999e-05, "loss": 0.028125829994678497, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02853, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.8889319896697998, "learning_rate": 9.992417937414932e-05, "loss": 0.036532819271087646, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03721, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.6546633839607239, "learning_rate": 9.99064004568618e-05, "loss": 0.0357094407081604, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.03635, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 4.966506481170654, "learning_rate": 9.988675280095074e-05, "loss": 0.03590286523103714, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.03656, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.9231100082397461, "learning_rate": 9.986523722403528e-05, "loss": 0.034722648561000824, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.03533, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.7877213954925537, "learning_rate": 9.984185462146642e-05, "loss": 0.04910588636994362, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.05033, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.5524486899375916, "learning_rate": 9.98166059662897e-05, "loss": 0.03699364885687828, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.03769, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.52, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.6449137926101685, "learning_rate": 9.978949230920472e-05, "loss": 0.02617681957781315, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02652, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.7289488911628723, "learning_rate": 9.976051477852141e-05, "loss": 0.03515414148569107, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03578, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.203433632850647, "learning_rate": 9.972967458011312e-05, "loss": 0.07415419071912766, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.07697, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.0358378887176514, "learning_rate": 9.96969729973664e-05, "loss": 0.05721193179488182, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.05888, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.6000136733055115, "learning_rate": 9.966241139112754e-05, "loss": 0.023098018020391464, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.02337, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 0.8115085959434509, "learning_rate": 9.96259911996461e-05, "loss": 0.020792683586478233, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02101, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.6471410989761353, "learning_rate": 9.958771393851491e-05, "loss": 0.05313863605260849, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.05458, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 1.2460017204284668, "learning_rate": 9.954758120060702e-05, "loss": 0.032700587064027786, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03324, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.9785034656524658, "learning_rate": 9.950559465600948e-05, "loss": 0.02041483484208584, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.02062, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 1.018216848373413, "learning_rate": 9.946175605195379e-05, "loss": 0.025861576199531555, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0262, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.8796634674072266, "learning_rate": 9.941606721274322e-05, "loss": 0.02234790101647377, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.0226, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 1.1162790060043335, "learning_rate": 9.936853003967685e-05, "loss": 0.03503534197807312, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03566, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.0729519128799438, "learning_rate": 9.93191465109705e-05, "loss": 0.022839998826384544, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.0231, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.0334558486938477, "learning_rate": 9.926791868167438e-05, "loss": 0.011986805126070976, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01206, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 2.2883787155151367, "learning_rate": 9.921484868358753e-05, "loss": 0.03625238314270973, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03692, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.2372729778289795, "learning_rate": 9.915993872516924e-05, "loss": 0.018473543226718903, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01865, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.7276994585990906, "learning_rate": 9.9103191091447e-05, "loss": 0.02411087229847908, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0244, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.47717246413230896, "learning_rate": 9.904460814392147e-05, "loss": 0.02103344537317753, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02126, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 1.5157725811004639, "learning_rate": 9.898419232046825e-05, "loss": 0.035099200904369354, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03572, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.7549039125442505, "learning_rate": 9.892194613523633e-05, "loss": 0.027264045551419258, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.02764, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.25944414734840393, "learning_rate": 9.885787217854357e-05, "loss": 0.009902503341436386, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00995, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.8252955079078674, "learning_rate": 9.879197311676887e-05, "loss": 0.035221487283706665, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.03585, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.6174755096435547, "learning_rate": 9.872425169224113e-05, "loss": 0.04023004323244095, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.04105, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 0.54942387342453, "learning_rate": 9.865471072312528e-05, "loss": 0.023006103932857513, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02327, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.20637722313404083, "learning_rate": 9.858335310330492e-05, "loss": 0.009584905579686165, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00963, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.4221399426460266, "learning_rate": 9.851018180226185e-05, "loss": 0.011705105192959309, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01177, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.3396678566932678, "learning_rate": 9.843519986495259e-05, "loss": 0.02154754102230072, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02178, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.31740137934684753, "learning_rate": 9.835841041168162e-05, "loss": 0.018234238028526306, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0184, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.5514931082725525, "learning_rate": 9.82798166379715e-05, "loss": 0.031676825135946274, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.03218, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 1.4310076236724854, "learning_rate": 9.819942181443002e-05, "loss": 0.06200280785560608, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.06397, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.8849724531173706, "learning_rate": 9.811722928661392e-05, "loss": 0.027368364855647087, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.02775, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.3798186779022217, "learning_rate": 9.803324247488975e-05, "loss": 0.005112533923238516, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00513, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.14075878262519836, "learning_rate": 9.794746487429161e-05, "loss": 0.004089841619133949, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.0041, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.6094908118247986, "learning_rate": 9.785990005437554e-05, "loss": 0.010526840575039387, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01058, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.6123790740966797, "learning_rate": 9.777055165907117e-05, "loss": 0.04000994935631752, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.04082, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 2.7964255809783936, "learning_rate": 9.767942340652993e-05, "loss": 0.014967570081353188, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01508, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.47, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.9038290977478027, "learning_rate": 9.758651908897035e-05, "loss": 0.03138715773820877, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.03188, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 1.3062785863876343, "learning_rate": 9.749184257252033e-05, "loss": 0.031881026923656464, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.03239, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.9979426860809326, "learning_rate": 9.739539779705614e-05, "loss": 0.023247070610523224, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.02352, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.752375602722168, "learning_rate": 9.729718877603861e-05, "loss": 0.02916880138218403, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0296, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.3952581882476807, "learning_rate": 9.719721959634592e-05, "loss": 0.03098556399345398, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03147, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.7812491059303284, "learning_rate": 9.709549441810375e-05, "loss": 0.01859249919652939, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01877, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.7877218127250671, "learning_rate": 9.699201747451195e-05, "loss": 0.02777542546391487, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02816, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.7160577774047852, "learning_rate": 9.688679307166854e-05, "loss": 0.021994777023792267, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.02224, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.6542000770568848, "learning_rate": 9.677982558839042e-05, "loss": 0.01505831815302372, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01517, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 1.1785330772399902, "learning_rate": 9.66711194760312e-05, "loss": 0.02272886410355568, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02299, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.7000851631164551, "learning_rate": 9.656067925829593e-05, "loss": 0.01005668193101883, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01011, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.7799171209335327, "learning_rate": 9.644850953105288e-05, "loss": 0.025370463728904724, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.0257, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.9042792916297913, "learning_rate": 9.633461496214225e-05, "loss": 0.016036491841077805, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01617, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.9807924032211304, "learning_rate": 9.621900029118195e-05, "loss": 0.03542560711503029, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03606, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.9542346596717834, "learning_rate": 9.610167032937036e-05, "loss": 0.011981204152107239, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01205, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.5268347263336182, "learning_rate": 9.598262995928611e-05, "loss": 0.015610314905643463, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01573, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.38, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.2429209202528, "learning_rate": 9.586188413468492e-05, "loss": 0.006768728140741587, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00679, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.6089392900466919, "learning_rate": 9.57394378802934e-05, "loss": 0.014144038781523705, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01424, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 1.6934500932693481, "learning_rate": 9.56152962916e-05, "loss": 0.043434858322143555, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.04439, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.5430171489715576, "learning_rate": 9.548946453464296e-05, "loss": 0.008121036924421787, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00815, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.5458353757858276, "learning_rate": 9.53619478457953e-05, "loss": 0.023594647645950317, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.02388, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.26810672879219055, "learning_rate": 9.523275153154695e-05, "loss": 0.0031822575256228447, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00319, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.6536190509796143, "learning_rate": 9.51018809682839e-05, "loss": 0.01843300648033619, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0186, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.46427056193351746, "learning_rate": 9.49693416020645e-05, "loss": 0.015746302902698517, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01587, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.3159849941730499, "learning_rate": 9.483513894839276e-05, "loss": 0.007879561744630337, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00791, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.6270001530647278, "learning_rate": 9.469927859198888e-05, "loss": 0.0081546725705266, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00819, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 1.2379077672958374, "learning_rate": 9.456176618655689e-05, "loss": 0.033253349363803864, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.03381, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.7684104442596436, "learning_rate": 9.442260745454927e-05, "loss": 0.02194179967045784, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.02218, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.3221273124217987, "learning_rate": 9.428180818692884e-05, "loss": 0.011272723786532879, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01134, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.31785956025123596, "learning_rate": 9.413937424292791e-05, "loss": 0.023024294525384903, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.02329, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.4135056436061859, "learning_rate": 9.399531154980424e-05, "loss": 0.016378924250602722, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01651, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.530357837677002, "learning_rate": 9.384962610259455e-05, "loss": 0.013228553347289562, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01332, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 1.592633605003357, "learning_rate": 9.370232396386494e-05, "loss": 0.042574502527713776, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.04349, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.4455731511116028, "learning_rate": 9.355341126345868e-05, "loss": 0.021547172218561172, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02178, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.380654513835907, "learning_rate": 9.340289419824107e-05, "loss": 0.01109155360609293, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.01115, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.6106314063072205, "learning_rate": 9.325077903184159e-05, "loss": 0.06568527221679688, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.06789, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.3074811100959778, "learning_rate": 9.30970720943932e-05, "loss": 0.026523005217313766, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.02688, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.43709999322891235, "learning_rate": 9.2941779782269e-05, "loss": 0.015758143737912178, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01588, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.23746491968631744, "learning_rate": 9.278490855781596e-05, "loss": 0.010707041248679161, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.01076, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.07, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.4756235182285309, "learning_rate": 9.262646494908604e-05, "loss": 0.027014464139938354, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.02738, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.29781007766723633, "learning_rate": 9.246645554956457e-05, "loss": 0.02757464163005352, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02796, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.5687547326087952, "learning_rate": 9.230488701789578e-05, "loss": 0.03793422505259514, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.03866, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.3516978621482849, "learning_rate": 9.214176607760577e-05, "loss": 0.00997837632894516, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01003, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.40119844675064087, "learning_rate": 9.197709951682268e-05, "loss": 0.02484101429581642, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02515, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 0.8507723808288574, "learning_rate": 9.181089418799428e-05, "loss": 0.031164126470685005, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.03165, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.3052947223186493, "learning_rate": 9.164315700760271e-05, "loss": 0.017849240452051163, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01801, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.8471056818962097, "learning_rate": 9.147389495587671e-05, "loss": 0.029755448922514915, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0302, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.91, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.3416641056537628, "learning_rate": 9.130311507650116e-05, "loss": 0.01015247218310833, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0102, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 3.0859341621398926, "learning_rate": 9.113082447632394e-05, "loss": 0.02973182685673237, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.03018, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.99, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 1.7546417713165283, "learning_rate": 9.09570303250602e-05, "loss": 0.015066434629261494, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01518, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.8502011895179749, "learning_rate": 9.078173985499394e-05, "loss": 0.01072525605559349, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01078, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 2.4968626499176025, "learning_rate": 9.060496036067713e-05, "loss": 0.03264159709215164, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.03318, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.36, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 2.175825834274292, "learning_rate": 9.042669919862615e-05, "loss": 0.03351624682545662, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.03408, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.7045714259147644, "learning_rate": 9.024696378701557e-05, "loss": 0.033460117876529694, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.03403, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.6932166814804077, "learning_rate": 9.006576160536948e-05, "loss": 0.020684774965047836, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.0209, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.8507452607154846, "learning_rate": 8.988310019425035e-05, "loss": 0.01671629212796688, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01686, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.252560019493103, "learning_rate": 8.969898715494506e-05, "loss": 0.0048630861565470695, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00487, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.42266198992729187, "learning_rate": 8.951343014914869e-05, "loss": 0.030870821326971054, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03135, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.6550277471542358, "learning_rate": 8.932643689864568e-05, "loss": 0.010384103283286095, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.01044, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.27519306540489197, "learning_rate": 8.913801518498845e-05, "loss": 0.006221175193786621, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00624, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.19, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.8020931482315063, "learning_rate": 8.894817284917364e-05, "loss": 0.0298006534576416, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.03025, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.6851024627685547, "learning_rate": 8.875691779131569e-05, "loss": 0.055303677916526794, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.05686, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.63, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.2830502390861511, "learning_rate": 8.856425797031829e-05, "loss": 0.00939229130744934, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00944, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.3299742341041565, "learning_rate": 8.837020140354295e-05, "loss": 0.013630540110170841, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.01372, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.2873094975948334, "learning_rate": 8.817475616647554e-05, "loss": 0.015454735606908798, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01557, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.3718639314174652, "learning_rate": 8.797793039239017e-05, "loss": 0.027186637744307518, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.02756, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.4399015009403229, "learning_rate": 8.777973227201069e-05, "loss": 0.01680174469947815, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01694, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.4798048138618469, "learning_rate": 8.758017005316988e-05, "loss": 0.017017925158143044, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01716, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.578261137008667, "learning_rate": 8.737925204046629e-05, "loss": 0.02735798805952072, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02774, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.37889009714126587, "learning_rate": 8.717698659491851e-05, "loss": 0.008906442672014236, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00895, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.3144233524799347, "learning_rate": 8.697338213361735e-05, "loss": 0.01913628540933132, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01932, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.3961825668811798, "learning_rate": 8.676844712937552e-05, "loss": 0.008896410465240479, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00894, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.24101771414279938, "learning_rate": 8.656219011037509e-05, "loss": 0.01150265522301197, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01157, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.27613529562950134, "learning_rate": 8.63546196598125e-05, "loss": 0.015537379309535027, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01566, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.8858233690261841, "learning_rate": 8.614574441554145e-05, "loss": 0.021157508715987206, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.02138, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.9801561236381531, "learning_rate": 8.593557306971349e-05, "loss": 0.01822391338646412, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01839, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.39710715413093567, "learning_rate": 8.572411436841618e-05, "loss": 0.0037306728772819042, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00374, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.9894429445266724, "learning_rate": 8.551137711130922e-05, "loss": 0.013201385736465454, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.01329, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 1.3725148439407349, "learning_rate": 8.529737015125824e-05, "loss": 0.00879655871540308, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00884, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.5669066309928894, "learning_rate": 8.508210239396639e-05, "loss": 0.015160775743424892, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01528, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.30306276679039, "learning_rate": 8.486558279760375e-05, "loss": 0.004290583077818155, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.0043, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 1.9637274742126465, "learning_rate": 8.464782037243449e-05, "loss": 0.011944068595767021, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01202, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 1.317379355430603, "learning_rate": 8.442882418044202e-05, "loss": 0.028496410697698593, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.02891, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 1.346110224723816, "learning_rate": 8.420860333495179e-05, "loss": 0.023191586136817932, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02346, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.20797358453273773, "learning_rate": 8.398716700025208e-05, "loss": 0.004703973885625601, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00472, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 1.0402941703796387, "learning_rate": 8.376452439121266e-05, "loss": 0.04912330210208893, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.05035, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 1.117400884628296, "learning_rate": 8.354068477290124e-05, "loss": 0.02096966654062271, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02119, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.17594203352928162, "learning_rate": 8.331565746019807e-05, "loss": 0.01026361994445324, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01032, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.33080288767814636, "learning_rate": 8.308945181740812e-05, "loss": 0.011099055409431458, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01116, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.38734516501426697, "learning_rate": 8.286207725787153e-05, "loss": 0.012774763628840446, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01286, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.3044201135635376, "learning_rate": 8.263354324357182e-05, "loss": 0.013715446926653385, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01381, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.2202962189912796, "learning_rate": 8.240385928474219e-05, "loss": 0.0097881481051445, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00984, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.19925367832183838, "learning_rate": 8.217303493946967e-05, "loss": 0.005558321718126535, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00557, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.48168525099754333, "learning_rate": 8.194107981329746e-05, "loss": 0.005777075886726379, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00579, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.2085031270980835, "learning_rate": 8.170800355882518e-05, "loss": 0.00324689457193017, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00325, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.29, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.3459969162940979, "learning_rate": 8.147381587530713e-05, "loss": 0.01567416824400425, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.0158, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.25320902466773987, "learning_rate": 8.123852650824877e-05, "loss": 0.012066018767654896, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01214, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.41334956884384155, "learning_rate": 8.100214524900103e-05, "loss": 0.004702077247202396, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00471, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.22800786793231964, "learning_rate": 8.076468193435301e-05, "loss": 0.002106434665620327, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00211, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.5788777470588684, "learning_rate": 8.052614644612253e-05, "loss": 0.008709074929356575, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00875, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.38009652495384216, "learning_rate": 8.028654871074489e-05, "loss": 0.003754644189029932, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00376, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.7109793424606323, "learning_rate": 8.004589869885986e-05, "loss": 0.0595184862613678, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.06133, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.46276912093162537, "learning_rate": 7.980420642489674e-05, "loss": 0.005838054232299328, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00586, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.5055245757102966, "learning_rate": 7.95614819466576e-05, "loss": 0.009807536378502846, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00986, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.9094300270080566, "learning_rate": 7.931773536489872e-05, "loss": 0.023495066910982132, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02377, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.8642670512199402, "learning_rate": 7.907297682291035e-05, "loss": 0.01972316950559616, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01992, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.3584756851196289, "learning_rate": 7.882721650609442e-05, "loss": 0.008019275963306427, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00805, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.8019735813140869, "learning_rate": 7.85804646415409e-05, "loss": 0.013642602600157261, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01374, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.44644683599472046, "learning_rate": 7.833273149760207e-05, "loss": 0.017135266214609146, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.01728, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.3008320927619934, "learning_rate": 7.808402738346527e-05, "loss": 0.028826219961047173, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02925, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.2755255401134491, "learning_rate": 7.783436264872382e-05, "loss": 0.006719534285366535, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00674, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.3671645224094391, "learning_rate": 7.758374768294647e-05, "loss": 0.009030526503920555, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00907, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.32322928309440613, "learning_rate": 7.733219291524489e-05, "loss": 0.01743132434785366, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01758, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.5090904831886292, "learning_rate": 7.707970881383977e-05, "loss": 0.02625729702413082, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.02661, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.2381606101989746, "learning_rate": 7.682630588562518e-05, "loss": 0.01408267393708229, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01418, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.32740435004234314, "learning_rate": 7.657199467573129e-05, "loss": 0.01734389364719391, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0175, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.33256295323371887, "learning_rate": 7.631678576708561e-05, "loss": 0.005804623942822218, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00582, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.31041377782821655, "learning_rate": 7.606068977997255e-05, "loss": 0.012088448740541935, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.01216, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.9913273453712463, "learning_rate": 7.580371737159148e-05, "loss": 0.00832087267190218, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00836, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.18643318116664886, "learning_rate": 7.554587923561324e-05, "loss": 0.0025948495604097843, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0026, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.5820580124855042, "learning_rate": 7.528718610173511e-05, "loss": 0.010679169557988644, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01074, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.76, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.3967525064945221, "learning_rate": 7.502764873523431e-05, "loss": 0.010236197151243687, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01029, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.4688413143157959, "learning_rate": 7.476727793652011e-05, "loss": 0.01159199420362711, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.01166, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.3899683952331543, "learning_rate": 7.450608454068415e-05, "loss": 0.007964255288243294, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.008, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.4655027687549591, "learning_rate": 7.424407941704987e-05, "loss": 0.0042363377287983894, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00425, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.6979988217353821, "learning_rate": 7.398127346871986e-05, "loss": 0.010636293329298496, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01069, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.21398359537124634, "learning_rate": 7.371767763212238e-05, "loss": 0.002193988999351859, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0022, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.7489000558853149, "learning_rate": 7.345330287655617e-05, "loss": 0.023018140345811844, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02329, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 1.0267404317855835, "learning_rate": 7.31881602037339e-05, "loss": 0.043418221175670624, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.04437, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.45164576172828674, "learning_rate": 7.29222606473245e-05, "loss": 0.008583104237914085, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00862, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.19363021850585938, "learning_rate": 7.265561527249383e-05, "loss": 0.005215784534811974, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00523, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.2953522503376007, "learning_rate": 7.238823517544436e-05, "loss": 0.00523486640304327, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00525, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.15, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.1751725971698761, "learning_rate": 7.212013148295333e-05, "loss": 0.003029226092621684, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00303, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.8862740397453308, "learning_rate": 7.185131535190975e-05, "loss": 0.01282970979809761, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01291, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.3649778366088867, "learning_rate": 7.158179796885005e-05, "loss": 0.008990784175693989, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00903, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.19683751463890076, "learning_rate": 7.131159054949273e-05, "loss": 0.008183285593986511, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00822, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.44093918800354004, "learning_rate": 7.104070433827139e-05, "loss": 0.017403926700353622, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01756, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.20664574205875397, "learning_rate": 7.076915060786705e-05, "loss": 0.006395455449819565, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00642, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.3645344376564026, "learning_rate": 7.049694065873882e-05, "loss": 0.01340278796851635, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01349, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.5502551198005676, "learning_rate": 7.022408581865382e-05, "loss": 0.014237520284950733, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01434, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.3804529905319214, "learning_rate": 6.99505974422157e-05, "loss": 0.015605336055159569, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.01573, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.39180418848991394, "learning_rate": 6.967648691039213e-05, "loss": 0.01032395288348198, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.01038, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.06915003061294556, "learning_rate": 6.940176563004123e-05, "loss": 0.0013830410316586494, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00138, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.5153525471687317, "learning_rate": 6.912644503343682e-05, "loss": 0.012091356329619884, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01216, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.31163349747657776, "learning_rate": 6.885053657779273e-05, "loss": 0.009590106084942818, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00964, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.1264515221118927, "learning_rate": 6.857405174478604e-05, "loss": 0.0028381722513586283, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00284, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.46084925532341003, "learning_rate": 6.82970020400792e-05, "loss": 0.012012304738163948, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01208, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.27550560235977173, "learning_rate": 6.801939899284132e-05, "loss": 0.004884413443505764, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.0049, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.022268518805503845, "learning_rate": 6.774125415526827e-05, "loss": 0.0004917756887152791, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00049, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.09633069485425949, "learning_rate": 6.746257910210214e-05, "loss": 0.002411237917840481, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00241, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.6407724022865295, "learning_rate": 6.718338543014937e-05, "loss": 0.00826188549399376, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.0083, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.77, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.05708516761660576, "learning_rate": 6.69036847577983e-05, "loss": 0.0006925716297701001, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00069, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.719925045967102, "learning_rate": 6.662348872453553e-05, "loss": 0.009982682764530182, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01003, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.5183490514755249, "learning_rate": 6.63428089904618e-05, "loss": 0.0033812751062214375, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00339, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.16121019423007965, "learning_rate": 6.60616572358065e-05, "loss": 0.0011752769351005554, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00118, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.15112850069999695, "learning_rate": 6.578004516044172e-05, "loss": 0.001926145050674677, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00193, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.05954829230904579, "learning_rate": 6.549798448339548e-05, "loss": 0.00034426318597979844, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00034, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.12370475381612778, "learning_rate": 6.521548694236384e-05, "loss": 0.0013169472804293036, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00132, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.08046429604291916, "learning_rate": 6.493256429322259e-05, "loss": 0.0007128622382879257, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00071, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.405707985162735, "learning_rate": 6.464922830953799e-05, "loss": 0.005891055800020695, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00591, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.8649346828460693, "learning_rate": 6.436549078207688e-05, "loss": 0.026538439095020294, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02689, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.1256963461637497, "learning_rate": 6.408136351831592e-05, "loss": 0.001377980108372867, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00138, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.20813098549842834, "learning_rate": 6.379685834195036e-05, "loss": 0.0014323488576337695, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00143, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.29230746626853943, "learning_rate": 6.351198709240186e-05, "loss": 0.004583702422678471, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00459, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.03, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.8741452097892761, "learning_rate": 6.32267616243259e-05, "loss": 0.008727950975298882, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00877, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.026653185486793518, "learning_rate": 6.294119380711849e-05, "loss": 0.00029235094552859664, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00029, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.9994991421699524, "learning_rate": 6.265529552442209e-05, "loss": 0.016165683045983315, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.0163, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.1544787436723709, "learning_rate": 6.236907867363127e-05, "loss": 0.0015394364017993212, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00154, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.26827776432037354, "learning_rate": 6.208255516539749e-05, "loss": 0.0028432600665837526, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00285, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.29710716009140015, "learning_rate": 6.179573692313344e-05, "loss": 0.006102443672716618, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00612, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.09595762938261032, "learning_rate": 6.150863588251694e-05, "loss": 0.0006361317355185747, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00064, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.49623093008995056, "learning_rate": 6.122126399099419e-05, "loss": 0.004170455504208803, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00418, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 3.445399045944214, "learning_rate": 6.0933633207282615e-05, "loss": 0.00955267809331417, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0096, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.3399350047111511, "learning_rate": 6.064575550087316e-05, "loss": 0.00742521695792675, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00745, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.21648308634757996, "learning_rate": 6.0357642851532245e-05, "loss": 0.003215834964066744, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00322, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.21981139481067657, "learning_rate": 6.0069307248803294e-05, "loss": 0.00506327161565423, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00508, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.81, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.5443705320358276, "learning_rate": 5.9780760691507635e-05, "loss": 0.0026045339182019234, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00261, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.3111158013343811, "learning_rate": 5.9492015187245334e-05, "loss": 0.004877505358308554, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00489, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.08662150055170059, "learning_rate": 5.920308275189541e-05, "loss": 0.001571443397551775, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00157, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.4410625994205475, "learning_rate": 5.8913975409115874e-05, "loss": 0.00803336687386036, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00807, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.3217575252056122, "learning_rate": 5.8624705189843395e-05, "loss": 0.0025593030732125044, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00256, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.25059089064598083, "learning_rate": 5.833528413179249e-05, "loss": 0.0016511641442775726, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00165, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.35431110858917236, "learning_rate": 5.80457242789548e-05, "loss": 0.010131891816854477, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01018, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.27414149045944214, "learning_rate": 5.77560376810977e-05, "loss": 0.002246221760287881, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00225, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.02704455517232418, "learning_rate": 5.7466236393263005e-05, "loss": 0.0003400572168175131, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00034, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.22877860069274902, "learning_rate": 5.717633247526522e-05, "loss": 0.004953057039529085, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00497, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.19363529980182648, "learning_rate": 5.688633799118971e-05, "loss": 0.0018539703451097012, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00186, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.17527954280376434, "learning_rate": 5.659626500889066e-05, "loss": 0.002147830091416836, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00215, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.233854278922081, "learning_rate": 5.6306125599488905e-05, "loss": 0.002005239948630333, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00201, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.92, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.2106197327375412, "learning_rate": 5.601593183686955e-05, "loss": 0.003330615581944585, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00334, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.15732808411121368, "learning_rate": 5.572569579717961e-05, "loss": 0.0022611869499087334, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00226, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.014531238935887814, "learning_rate": 5.543542955832538e-05, "loss": 0.00019866356160491705, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0002, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.3887158930301666, "learning_rate": 5.514514519946986e-05, "loss": 0.004210400395095348, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00422, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.41414764523506165, "learning_rate": 5.485485480053015e-05, "loss": 0.008840526454150677, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00888, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.74, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.32716286182403564, "learning_rate": 5.4564570441674645e-05, "loss": 0.004206635989248753, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00422, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.18061913549900055, "learning_rate": 5.42743042028204e-05, "loss": 0.0021151078399270773, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00212, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.36, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.2585257887840271, "learning_rate": 5.3984068163130464e-05, "loss": 0.004433964844793081, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00444, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.2843065559864044, "learning_rate": 5.369387440051111e-05, "loss": 0.0042855204083025455, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00429, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.26981157064437866, "learning_rate": 5.340373499110935e-05, "loss": 0.0018633807776495814, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00187, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 1.0496301651000977, "learning_rate": 5.3113662008810304e-05, "loss": 0.011695988476276398, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.01176, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.46307092905044556, "learning_rate": 5.282366752473479e-05, "loss": 0.0025333091616630554, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00254, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.038470737636089325, "learning_rate": 5.2533763606737005e-05, "loss": 0.0004884620429947972, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00049, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.14, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.027404172345995903, "learning_rate": 5.224396231890232e-05, "loss": 7.704585732426494e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00008, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.2805972397327423, "learning_rate": 5.195427572104522e-05, "loss": 0.012020820751786232, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01209, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.0811084434390068, "learning_rate": 5.166471586820751e-05, "loss": 0.0007037121686153114, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.0007, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.1958155483007431, "learning_rate": 5.1375294810156615e-05, "loss": 0.0015244274400174618, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00153, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.18918555974960327, "learning_rate": 5.1086024590884144e-05, "loss": 0.0012832059292122722, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00128, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.264037162065506, "learning_rate": 5.079691724810461e-05, "loss": 0.003356513101607561, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00336, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.014500330202281475, "learning_rate": 5.0507984812754684e-05, "loss": 0.00010566271521383896, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00011, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.011936323717236519, "learning_rate": 5.021923930849237e-05, "loss": 0.00016189023153856397, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00016, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.27654555439949036, "learning_rate": 4.99306927511967e-05, "loss": 0.0036926029715687037, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.0037, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 2.0797948837280273, "learning_rate": 4.964235714846775e-05, "loss": 0.008603260852396488, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00864, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.013610179536044598, "learning_rate": 4.9354244499126866e-05, "loss": 0.00013845613284502178, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00014, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.29, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.26970794796943665, "learning_rate": 4.90663667927174e-05, "loss": 0.002778912428766489, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00278, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.10973677039146423, "learning_rate": 4.877873600900581e-05, "loss": 0.0016775373369455338, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00168, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.74, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.12009457498788834, "learning_rate": 4.849136411748306e-05, "loss": 0.0014326423406600952, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00143, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.4458884298801422, "learning_rate": 4.8204263076866574e-05, "loss": 0.006848520133644342, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00687, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.015622676350176334, "learning_rate": 4.791744483460251e-05, "loss": 0.00019457696180325001, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00019, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.026380350813269615, "learning_rate": 4.7630921326368736e-05, "loss": 0.00022359384456649423, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00022, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.98, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.022433504462242126, "learning_rate": 4.7344704475577916e-05, "loss": 0.00032186246244236827, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00032, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.2329663187265396, "learning_rate": 4.705880619288153e-05, "loss": 0.0016043068608269095, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00161, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.97, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.25109946727752686, "learning_rate": 4.677323837567412e-05, "loss": 0.003007173538208008, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00301, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.002927472349256277, "learning_rate": 4.6488012907598146e-05, "loss": 5.627587961498648e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00006, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.29643774032592773, "learning_rate": 4.620314165804964e-05, "loss": 0.002507295925170183, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00251, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.4524941146373749, "learning_rate": 4.591863648168407e-05, "loss": 0.006622933316975832, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00664, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.012826545163989067, "learning_rate": 4.5634509217923135e-05, "loss": 0.00011967686441494152, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00012, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.16463270783424377, "learning_rate": 4.535077169046201e-05, "loss": 0.0027581623289734125, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00276, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.852912962436676, "learning_rate": 4.506743570677743e-05, "loss": 0.013197163119912148, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01328, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.18, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.3576260507106781, "learning_rate": 4.478451305763618e-05, "loss": 0.0037953476421535015, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.0038, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.002030731877312064, "learning_rate": 4.450201551660454e-05, "loss": 4.460547643247992e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00004, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.010063775815069675, "learning_rate": 4.4219954839558276e-05, "loss": 0.00010233936336589977, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0001, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.5394333600997925, "learning_rate": 4.393834276419352e-05, "loss": 0.006385323125869036, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00641, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.28, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.3952396810054779, "learning_rate": 4.36571910095382e-05, "loss": 0.003885653568431735, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00389, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.0734419971704483, "learning_rate": 4.337651127546448e-05, "loss": 0.0006130565307103097, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00061, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.006736129987984896, "learning_rate": 4.3096315242201736e-05, "loss": 6.332539487630129e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00006, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.13015973567962646, "learning_rate": 4.2816614569850635e-05, "loss": 0.0008574441890232265, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00086, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.3079664409160614, "learning_rate": 4.2537420897897864e-05, "loss": 0.003037002868950367, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00304, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.2179604470729828, "learning_rate": 4.225874584473174e-05, "loss": 0.002208883408457041, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00221, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.13965938985347748, "learning_rate": 4.19806010071587e-05, "loss": 0.0025114677846431732, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00251, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.29029563069343567, "learning_rate": 4.170299795992081e-05, "loss": 0.0038156243972480297, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00382, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.07329453527927399, "learning_rate": 4.142594825521398e-05, "loss": 0.0007944530807435513, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00079, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.1327333301305771, "learning_rate": 4.114946342220728e-05, "loss": 0.0012795457150787115, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00128, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.13315396010875702, "learning_rate": 4.087355496656321e-05, "loss": 0.000721147283911705, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00072, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.08956572413444519, "learning_rate": 4.05982343699588e-05, "loss": 0.000565249240025878, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00057, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.16947047412395477, "learning_rate": 4.0323513089607876e-05, "loss": 0.0015017908299341798, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0015, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.16568097472190857, "learning_rate": 4.004940255778431e-05, "loss": 0.0005019927048124373, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0005, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.21880680322647095, "learning_rate": 3.977591418134619e-05, "loss": 0.006334515754133463, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00635, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.02908463403582573, "learning_rate": 3.95030593412612e-05, "loss": 0.00029721265309490263, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.0003, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.00777435814961791, "learning_rate": 3.923084939213296e-05, "loss": 9.655172470957041e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.0001, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.7834826707839966, "learning_rate": 3.895929566172861e-05, "loss": 0.013336087577044964, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01343, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 2.1480355262756348, "learning_rate": 3.868840945050728e-05, "loss": 0.010132654570043087, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.01018, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.48, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.006272666621953249, "learning_rate": 3.841820203114995e-05, "loss": 6.751603359589353e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00007, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.008783373050391674, "learning_rate": 3.814868464809027e-05, "loss": 0.00010338950960431248, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0001, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.4850597679615021, "learning_rate": 3.787986851704667e-05, "loss": 0.009795165620744228, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00984, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.03123776614665985, "learning_rate": 3.7611764824555654e-05, "loss": 0.0002967835171148181, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0003, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.11037160456180573, "learning_rate": 3.734438472750619e-05, "loss": 0.001959763467311859, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00196, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.04463181644678116, "learning_rate": 3.707773935267552e-05, "loss": 0.00027573859551921487, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00028, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.010778462514281273, "learning_rate": 3.68118397962661e-05, "loss": 0.00020475013297982514, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.0002, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.07772523909807205, "learning_rate": 3.654669712344384e-05, "loss": 0.0009753478225320578, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00098, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.61, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.1604541838169098, "learning_rate": 3.628232236787763e-05, "loss": 0.00197398429736495, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00198, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.1145077496767044, "learning_rate": 3.6018726531280144e-05, "loss": 0.0007730089128017426, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00077, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.0728338360786438, "learning_rate": 3.575592058295017e-05, "loss": 0.0006850542267784476, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00069, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.0026529019232839346, "learning_rate": 3.549391545931585e-05, "loss": 5.326753307599574e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00005, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.0034034878481179476, "learning_rate": 3.5232722063479914e-05, "loss": 7.191763143055141e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00007, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.021720819175243378, "learning_rate": 3.49723512647657e-05, "loss": 0.0002643875777721405, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00026, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.04574006050825119, "learning_rate": 3.471281389826491e-05, "loss": 0.0005244454368948936, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00052, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.030409149825572968, "learning_rate": 3.4454120764386764e-05, "loss": 0.00025720795383676887, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00026, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.06700447201728821, "learning_rate": 3.4196282628408526e-05, "loss": 0.000701534969266504, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0007, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.013228384777903557, "learning_rate": 3.3939310220027456e-05, "loss": 0.00022312210057862103, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00022, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.004159604664891958, "learning_rate": 3.3683214232914404e-05, "loss": 4.1192961361957714e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00004, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.018442818894982338, "learning_rate": 3.342800532426873e-05, "loss": 0.00023038491781335324, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00023, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.2153363674879074, "learning_rate": 3.317369411437484e-05, "loss": 0.0022433821577578783, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00225, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.13423292338848114, "learning_rate": 3.292029118616024e-05, "loss": 0.0012429120251908898, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00124, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.2673902213573456, "learning_rate": 3.266780708475511e-05, "loss": 0.004559407941997051, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00457, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.05549699440598488, "learning_rate": 3.241625231705354e-05, "loss": 0.00036612458643503487, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00037, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.021343596279621124, "learning_rate": 3.216563735127618e-05, "loss": 0.000189846774446778, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00019, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.021605251356959343, "learning_rate": 3.191597261653475e-05, "loss": 0.00019897540914826095, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0002, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.7681000232696533, "learning_rate": 3.166726850239794e-05, "loss": 0.006137716118246317, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00616, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.016124369576573372, "learning_rate": 3.141953535845912e-05, "loss": 0.0001589412277098745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00016, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.785254955291748, "learning_rate": 3.11727834939056e-05, "loss": 0.003530982183292508, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00354, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.13654708862304688, "learning_rate": 3.092702317708967e-05, "loss": 0.0024519735015928745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00245, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.006263244431465864, "learning_rate": 3.0682264635101276e-05, "loss": 9.283063263865188e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00009, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.3070363998413086, "learning_rate": 3.0438518053342407e-05, "loss": 0.0014467618893831968, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00145, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.02972697652876377, "learning_rate": 3.0195793575103266e-05, "loss": 0.00014360339264385402, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00014, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.23, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.10037615895271301, "learning_rate": 2.9954101301140146e-05, "loss": 0.0009620100609026849, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00096, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.30600103735923767, "learning_rate": 2.9713451289255123e-05, "loss": 0.002314475830644369, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00232, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.012774504721164703, "learning_rate": 2.9473853553877484e-05, "loss": 0.00017401829245500267, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00017, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.011553168296813965, "learning_rate": 2.9235318065647e-05, "loss": 0.00013747724005952477, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00014, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.00984975602477789, "learning_rate": 2.8997854750998964e-05, "loss": 0.00014798202028032392, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00015, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.038600511848926544, "learning_rate": 2.8761473491751258e-05, "loss": 0.0003391751670278609, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00034, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.06175951659679413, "learning_rate": 2.8526184124692883e-05, "loss": 0.00036634516436606646, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00037, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.36, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.14624406397342682, "learning_rate": 2.829199644117484e-05, "loss": 0.001556510105729103, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00156, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.16718840599060059, "learning_rate": 2.8058920186702553e-05, "loss": 0.0022887527011334896, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00229, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.023151922971010208, "learning_rate": 2.782696506053033e-05, "loss": 0.00016491231508553028, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00016, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.020919568836688995, "learning_rate": 2.7596140715257824e-05, "loss": 9.486666385782883e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00009, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.012602157890796661, "learning_rate": 2.7366456756428184e-05, "loss": 0.00012925526243634522, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00013, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.024951156228780746, "learning_rate": 2.7137922742128486e-05, "loss": 0.00024624247453175485, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00025, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.06787644326686859, "learning_rate": 2.691054818259188e-05, "loss": 0.0003629435086622834, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00036, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.20991812646389008, "learning_rate": 2.6684342539801933e-05, "loss": 0.0017672169487923384, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00177, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.01873696595430374, "learning_rate": 2.645931522709877e-05, "loss": 0.00012325140414759517, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00012, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.1211574450135231, "learning_rate": 2.6235475608787365e-05, "loss": 0.0018154771532863379, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00182, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 1.559299349784851, "learning_rate": 2.6012832999747916e-05, "loss": 0.003006167709827423, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00301, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.29, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.8308534622192383, "learning_rate": 2.579139666504821e-05, "loss": 0.025105351582169533, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02542, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.04862397164106369, "learning_rate": 2.557117581955798e-05, "loss": 0.0003772217023652047, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00038, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.5120987892150879, "learning_rate": 2.5352179627565532e-05, "loss": 0.005810985341668129, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00583, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.011630048044025898, "learning_rate": 2.5134417202396277e-05, "loss": 0.00011662894394248724, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00012, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.14042069017887115, "learning_rate": 2.491789760603361e-05, "loss": 0.0009937528520822525, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00099, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.017578125, "learning_rate": 2.4702629848741764e-05, "loss": 0.00021238779299892485, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00021, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.2494453489780426, "learning_rate": 2.4488622888690785e-05, "loss": 0.0006668390706181526, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00067, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.008109943941235542, "learning_rate": 2.427588563158384e-05, "loss": 8.694575808476657e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00009, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.007187106646597385, "learning_rate": 2.406442693028651e-05, "loss": 8.913000783650205e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00009, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.02686469443142414, "learning_rate": 2.3854255584458547e-05, "loss": 0.00023225189943332225, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00023, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.58, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.13875436782836914, "learning_rate": 2.3645380340187508e-05, "loss": 0.0008088515605777502, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00081, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.11577730625867844, "learning_rate": 2.3437809889624914e-05, "loss": 0.0013657942181453109, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00137, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.18, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.002642127452418208, "learning_rate": 2.3231552870624487e-05, "loss": 5.818655699840747e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00006, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.18635080754756927, "learning_rate": 2.3026617866382657e-05, "loss": 0.0012267834972590208, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00123, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.20455701649188995, "learning_rate": 2.2823013405081507e-05, "loss": 0.0012633336009457707, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00126, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.0644175186753273, "learning_rate": 2.2620747959533722e-05, "loss": 0.0003286988358013332, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00033, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.67, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.02383357472717762, "learning_rate": 2.2419829946830123e-05, "loss": 0.00017207785276696086, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00017, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.07526605576276779, "learning_rate": 2.2220267727989325e-05, "loss": 0.0005510843475349247, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00055, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.21450117230415344, "learning_rate": 2.202206960760984e-05, "loss": 0.0018159281462430954, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00182, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.07652191072702408, "learning_rate": 2.182524383352446e-05, "loss": 0.0007015472510829568, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0007, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.39340224862098694, "learning_rate": 2.1629798596457056e-05, "loss": 0.013283845968544483, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01337, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.009849383495748043, "learning_rate": 2.1435742029681725e-05, "loss": 0.0001477572222938761, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00015, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.015083015896379948, "learning_rate": 2.124308220868431e-05, "loss": 0.00011877882207045332, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00012, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.2895221412181854, "learning_rate": 2.105182715082638e-05, "loss": 0.002705915132537484, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00271, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.07597485929727554, "learning_rate": 2.0861984815011552e-05, "loss": 0.0007814620621502399, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00078, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.015677735209465027, "learning_rate": 2.0673563101354323e-05, "loss": 0.00012036257248837501, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00012, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.31561553478240967, "learning_rate": 2.0486569850851317e-05, "loss": 0.003808199195191264, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00382, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.01987501233816147, "learning_rate": 2.0301012845054956e-05, "loss": 8.235283894464374e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00008, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.47, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.0026104131247848272, "learning_rate": 2.011689980574966e-05, "loss": 5.0939768698299304e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00005, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.035882413387298584, "learning_rate": 1.993423839463052e-05, "loss": 0.00027505189063958824, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00028, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.02834615297615528, "learning_rate": 1.975303621298445e-05, "loss": 0.00017529382603242993, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00018, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.03981553018093109, "learning_rate": 1.957330080137385e-05, "loss": 0.00036935933167114854, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00037, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.1716083586215973, "learning_rate": 1.9395039639322864e-05, "loss": 0.000605978537350893, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00061, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.1408384144306183, "learning_rate": 1.9218260145006073e-05, "loss": 0.0006301657413132489, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00063, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.009179619140923023, "learning_rate": 1.904296967493982e-05, "loss": 7.324818579945713e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00007, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 3.745335578918457, "learning_rate": 1.8869175523676064e-05, "loss": 0.0016359509900212288, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00164, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.015083672478795052, "learning_rate": 1.869688492349885e-05, "loss": 0.00010518422641325742, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00011, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.005932419560849667, "learning_rate": 1.85261050441233e-05, "loss": 6.893977842992172e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00007, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 1.124258279800415, "learning_rate": 1.8356842992397304e-05, "loss": 0.018128983676433563, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01829, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.049792855978012085, "learning_rate": 1.8189105812005714e-05, "loss": 0.0003060678718611598, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00031, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.005102514289319515, "learning_rate": 1.802290048317732e-05, "loss": 0.00010844768257811666, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00011, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.007244511041790247, "learning_rate": 1.785823392239424e-05, "loss": 4.7965062549337745e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00005, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.12413360923528671, "learning_rate": 1.7695112982104225e-05, "loss": 0.0016369106015190482, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00164, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.05764605477452278, "learning_rate": 1.7533544450435433e-05, "loss": 0.0004322348104324192, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00043, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.0036381951067596674, "learning_rate": 1.7373535050913946e-05, "loss": 4.567591895465739e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00005, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.89, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.05374005064368248, "learning_rate": 1.721509144218405e-05, "loss": 0.0004705698520410806, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00047, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.0051624649204313755, "learning_rate": 1.705822021773101e-05, "loss": 7.183889829320833e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00007, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.22548705339431763, "learning_rate": 1.69029279056068e-05, "loss": 0.008193974383175373, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00823, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.1575491726398468, "learning_rate": 1.6749220968158415e-05, "loss": 0.0014256442664191127, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00143, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.043032146990299225, "learning_rate": 1.659710580175893e-05, "loss": 0.0003647786215879023, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00036, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.14, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.14046095311641693, "learning_rate": 1.644658873654133e-05, "loss": 0.0009069207590073347, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00091, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.0065954639576375484, "learning_rate": 1.629767603613508e-05, "loss": 0.0001061344300978817, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00011, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.040331821888685226, "learning_rate": 1.615037389740547e-05, "loss": 0.00044135001371614635, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00044, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.08710307627916336, "learning_rate": 1.600468845019576e-05, "loss": 0.0005320543423295021, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00053, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.014165140688419342, "learning_rate": 1.5860625757072092e-05, "loss": 9.817717364057899e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.0001, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.02613850124180317, "learning_rate": 1.571819181307116e-05, "loss": 4.4414657168090343e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00004, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.03678151220083237, "learning_rate": 1.557739254545075e-05, "loss": 0.00029438931960612535, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00029, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.73, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.007830196060240269, "learning_rate": 1.543823381344311e-05, "loss": 8.303801587317139e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00008, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.16, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.01465073972940445, "learning_rate": 1.5300721408011114e-05, "loss": 0.00018769476446323097, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00019, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.015662547200918198, "learning_rate": 1.5164861051607254e-05, "loss": 0.00017137316172011197, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00017, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.018299240618944168, "learning_rate": 1.5030658397935521e-05, "loss": 0.0001814981224015355, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00018, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.007216451223939657, "learning_rate": 1.4898119031716104e-05, "loss": 0.00011987227480858564, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00012, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.007300080265849829, "learning_rate": 1.476724846845306e-05, "loss": 0.00012195605086162686, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00012, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.012514130212366581, "learning_rate": 1.463805215420471e-05, "loss": 0.00016039583715610206, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00016, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.04540836066007614, "learning_rate": 1.451053546535705e-05, "loss": 0.0002649548987392336, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00026, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.014625420793890953, "learning_rate": 1.438470370840001e-05, "loss": 0.0001701970468275249, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00017, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.7968173027038574, "learning_rate": 1.4260562119706606e-05, "loss": 0.0011398601345717907, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00114, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.009781846776604652, "learning_rate": 1.413811586531508e-05, "loss": 0.00015647160762455314, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00016, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.7, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.714146614074707, "learning_rate": 1.4017370040713884e-05, "loss": 0.005084811709821224, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.0051, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 1.0036110877990723, "learning_rate": 1.3898329670629645e-05, "loss": 0.004250080790370703, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00426, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 21.0, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.10003674030303955, "learning_rate": 1.3780999708818058e-05, "loss": 0.0009638892370276153, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00096, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.0030406571459025145, "learning_rate": 1.3665385037857758e-05, "loss": 5.636852074530907e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00006, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.09735707938671112, "learning_rate": 1.3551490468947126e-05, "loss": 0.0006098074954934418, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00061, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.4188733398914337, "learning_rate": 1.3439320741704075e-05, "loss": 0.0039583779871463776, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00397, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.19352847337722778, "learning_rate": 1.3328880523968808e-05, "loss": 0.0014241538010537624, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00143, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.035671524703502655, "learning_rate": 1.3220174411609587e-05, "loss": 0.000245939678279683, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00025, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.024772340431809425, "learning_rate": 1.3113206928331471e-05, "loss": 0.0002784319513011724, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00028, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.73, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.02847646363079548, "learning_rate": 1.300798252548806e-05, "loss": 0.00016053173749241978, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00016, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.2864530682563782, "learning_rate": 1.2904505581896265e-05, "loss": 0.011202618479728699, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01127, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.009411647915840149, "learning_rate": 1.2802780403654082e-05, "loss": 0.00015762346447445452, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00016, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.0063218348659574986, "learning_rate": 1.2702811223961408e-05, "loss": 0.00010150793968932703, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0001, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.06979432702064514, "learning_rate": 1.2604602202943861e-05, "loss": 0.0002767994883470237, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00028, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.2682717740535736, "learning_rate": 1.2508157427479686e-05, "loss": 0.004307486116886139, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00432, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.4057947099208832, "learning_rate": 1.2413480911029655e-05, "loss": 0.004828420002013445, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00484, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.1590396612882614, "learning_rate": 1.2320576593470082e-05, "loss": 0.0007102082017809153, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00071, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 1.4085886478424072, "learning_rate": 1.2229448340928828e-05, "loss": 0.007242613006383181, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00727, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.02071862854063511, "learning_rate": 1.2140099945624458e-05, "loss": 0.00016221540863625705, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00016, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.025059983134269714, "learning_rate": 1.205253512570841e-05, "loss": 0.0003604017256293446, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00036, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.021521225571632385, "learning_rate": 1.1966757525110255e-05, "loss": 0.0002753809676505625, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00028, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.85, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.906055212020874, "learning_rate": 1.1882770713386095e-05, "loss": 0.003639915259554982, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00365, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.010934324003756046, "learning_rate": 1.180057818556998e-05, "loss": 6.660047802142799e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00007, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.12190091609954834, "learning_rate": 1.1720183362028494e-05, "loss": 0.0009676986373960972, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00097, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.015077483840286732, "learning_rate": 1.1641589588318387e-05, "loss": 0.00019447231898084283, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00019, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.004038236569613218, "learning_rate": 1.1564800135047418e-05, "loss": 4.2042258428409696e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00004, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.007483999244868755, "learning_rate": 1.148981819773816e-05, "loss": 0.00010619144450174645, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00011, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.15816745162010193, "learning_rate": 1.1416646896695086e-05, "loss": 0.0007582007092423737, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00076, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.021560903638601303, "learning_rate": 1.1345289276874717e-05, "loss": 0.0002545159077271819, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00025, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.1038987785577774, "learning_rate": 1.1275748307758873e-05, "loss": 0.0006665613618679345, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00067, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.16617074608802795, "learning_rate": 1.1208026883231147e-05, "loss": 0.0012382060522213578, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00124, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.0028197565115988255, "learning_rate": 1.1142127821456433e-05, "loss": 6.588864926015958e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00007, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.14940476417541504, "learning_rate": 1.1078053864763674e-05, "loss": 0.001126313814893365, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00113, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.5700771808624268, "learning_rate": 1.1015807679531756e-05, "loss": 0.004954281263053417, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00497, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.01477541122585535, "learning_rate": 1.0955391856078528e-05, "loss": 0.00015685860125813633, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00016, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.3097807466983795, "learning_rate": 1.0896808908553007e-05, "loss": 0.003279704600572586, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00329, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.4035567045211792, "learning_rate": 1.0840061274830763e-05, "loss": 0.005543005187064409, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00556, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.009225419722497463, "learning_rate": 1.0785151316412473e-05, "loss": 0.00010986075358232483, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00011, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.11226324737071991, "learning_rate": 1.0732081318325639e-05, "loss": 0.0008486650185659528, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00085, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.028836509212851524, "learning_rate": 1.0680853489029501e-05, "loss": 0.0002577693958301097, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00026, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.01875106431543827, "learning_rate": 1.0631469960323152e-05, "loss": 0.0001413324207533151, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00014, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.59, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.09526877850294113, "learning_rate": 1.0583932787256783e-05, "loss": 0.0010389768285676837, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00104, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.03344951942563057, "learning_rate": 1.0538243948046206e-05, "loss": 0.0003195603203494102, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00032, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.009598338976502419, "learning_rate": 1.0494405343990523e-05, "loss": 0.00011877785436809063, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00012, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.017349474132061005, "learning_rate": 1.0452418799392985e-05, "loss": 0.0001580297393957153, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00016, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.11384257674217224, "learning_rate": 1.0412286061485102e-05, "loss": 0.000755391432903707, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00076, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.009088551625609398, "learning_rate": 1.03740088003539e-05, "loss": 9.848400804912671e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.0001, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.58, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.03330879658460617, "learning_rate": 1.0337588608872463e-05, "loss": 0.00027810881147161126, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00028, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.05, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.18602754175662994, "learning_rate": 1.0303027002633622e-05, "loss": 0.0011074378853663802, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00111, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.012933547608554363, "learning_rate": 1.0270325419886884e-05, "loss": 9.155971929430962e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00009, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.07235568016767502, "learning_rate": 1.0239485221478599e-05, "loss": 0.0005825384287163615, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00058, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.2644723057746887, "learning_rate": 1.0210507690795292e-05, "loss": 0.0021688751876354218, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00217, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.05110086500644684, "learning_rate": 1.0183394033710305e-05, "loss": 0.0006004978204146028, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.0006, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.04905666783452034, "learning_rate": 1.0158145378533583e-05, "loss": 0.0003130019176751375, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00031, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.057489458471536636, "learning_rate": 1.0134762775964726e-05, "loss": 0.0004319213912822306, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00043, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.03, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.016210099682211876, "learning_rate": 1.0113247199049278e-05, "loss": 0.00013349046639632434, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00013, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.13936659693717957, "learning_rate": 1.0093599543138205e-05, "loss": 0.001083284616470337, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00108, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.19970905780792236, "learning_rate": 1.0075820625850675e-05, "loss": 0.0016534151509404182, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00165, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.4001736640930176, "learning_rate": 1.0059911187040013e-05, "loss": 0.0003034809196833521, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0003, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.023092353716492653, "learning_rate": 1.0045871888762893e-05, "loss": 0.0002565526228863746, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00026, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.013612436130642891, "learning_rate": 1.003370331525184e-05, "loss": 0.00016319255519192666, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00016, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.00489874416962266, "learning_rate": 1.002340597289085e-05, "loss": 6.14683231106028e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00006, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.03315464407205582, "learning_rate": 1.0014980290194387e-05, "loss": 0.0005261984770186245, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00053, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.007941976189613342, "learning_rate": 1.0008426617789489e-05, "loss": 0.0001453919685445726, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00015, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.09071626514196396, "learning_rate": 1.0003745228401215e-05, "loss": 0.0007300403667613864, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00073, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.4664659798145294, "learning_rate": 1.0000936316841296e-05, "loss": 0.0027483284939080477, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00275, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.98, "tokens/trainable": 232756} diff --git a/gemma3_12b_19m/coin/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_19m/coin/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbf7b7250cfa6d59249952d659e0e2a8a755ba77 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/coin/training/aft/mixed_coin/train.log b/gemma3_12b_19m/coin/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..90aa07d96347befe4e2cf0f7a987b13a80b9ef22 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 03:13:42,597] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 03:13:44.386000 111081 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:13:44.412000 111081 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 03:13:54,187] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 03:13:56.823000 112121 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:13:56.843000 112121 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 03:13:59,647] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 03:13:59,650] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 03:13:59,650] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 03:13:59,677] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 03:13:59,733] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 03:13:59,733] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 03:13:59,805] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 03:14:02,395] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/prepared/93313960e89195a0ef5e31d914ba563d +[2026-09-02 03:14:02,395] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 03:14:02,395] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 32348.58 examples/s] Generating train split: 8002 examples [00:00, 30846.17 examples/s] Generating train split: 8192 examples [00:00, 30349.61 examples/s] +[2026-09-02 03:14:03,707] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 03:14:03,710] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4318.94 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 20424.99 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.988000 112939 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.988000 112935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.993000 112946 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:02.994000 112940 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.009000 112939 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.010000 112935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.015000 112940 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.017000 112942 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.039000 112942 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.041000 112936 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.041000 112937 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.063000 112936 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.063000 112937 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.160000 112941 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.181000 112941 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.232000 112938 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 03:15:03.253000 112938 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:12<01:30, 79.31 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:12<01:30, 79.31 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:12<01:17, 79.31 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:12<01:04, 79.31 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:12<00:51, 79.31 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:12<00:38, 79.31 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:12<00:25, 79.31 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:12<00:12, 79.31 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 79.31 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:14<00:00, 582.63 examples/s] +[2026-09-02 03:15:08,050] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 03:15:17,440] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T04:30:41+00:00", + "resolved_config_sha256": "dbd729363483dc41dd5df1a0c1913d4c2b1c0d43c7f1ca619ddcf668beb212da", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "39afa0466aa7933f516f562ef67e39d2661e7e27346461519b7bfd782b0be953", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/coin/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "230ebeaf17fc03afc841145b6c664575522ac3d77117d1a0c232fe4d480eba91", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/coin/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_19m/coin/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..c4c8cbe84cf6a688fa3796f209a9ddc5b64389a8 --- /dev/null +++ b/gemma3_12b_19m/coin/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 0.9863632321357727, "learning_rate": 0.0, "loss": 0.1404155194759369, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.15075, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.3209733963012695, "learning_rate": 4.000000000000001e-06, "loss": 0.14079797267913818, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.15119, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.73, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.4731794595718384, "learning_rate": 8.000000000000001e-06, "loss": 0.13725724816322327, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.14712, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.4122910499572754, "learning_rate": 1.2e-05, "loss": 0.13823354244232178, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.14824, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.56, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 0.9519411325454712, "learning_rate": 1.6000000000000003e-05, "loss": 0.13145267963409424, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.14048, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 7.885478496551514, "learning_rate": 2e-05, "loss": 0.12285187840461731, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.13072, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.2108865976333618, "learning_rate": 2.4e-05, "loss": 0.11208675801753998, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.11861, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 2.0023443698883057, "learning_rate": 2.8000000000000003e-05, "loss": 0.11747317016124725, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.12465, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 3.831850051879883, "learning_rate": 3.2000000000000005e-05, "loss": 0.07918262481689453, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.0824, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.9498860836029053, "learning_rate": 3.6e-05, "loss": 0.10384620726108551, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.10943, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 3.1142194271087646, "learning_rate": 4e-05, "loss": 0.1268633008003235, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.13526, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 5.9658098220825195, "learning_rate": 4.4000000000000006e-05, "loss": 0.11020125448703766, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.1165, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 2.6365761756896973, "learning_rate": 4.8e-05, "loss": 0.06518758833408356, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.06736, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 1.9069676399230957, "learning_rate": 5.2000000000000004e-05, "loss": 0.06808874756097794, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.07046, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 3.132420063018799, "learning_rate": 5.6000000000000006e-05, "loss": 0.0735839307308197, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.07636, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.6334996223449707, "learning_rate": 6e-05, "loss": 0.07751709222793579, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.0806, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.79032301902771, "learning_rate": 6.400000000000001e-05, "loss": 0.05012615770101547, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0514, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.0421910285949707, "learning_rate": 6.800000000000001e-05, "loss": 0.03593416139483452, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03659, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.2129048109054565, "learning_rate": 7.2e-05, "loss": 0.032803528010845184, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.03335, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 1.2520902156829834, "learning_rate": 7.6e-05, "loss": 0.03146881237626076, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.03197, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.8274304270744324, "learning_rate": 8e-05, "loss": 0.027464278042316437, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02784, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.580518364906311, "learning_rate": 8.4e-05, "loss": 0.03887976333498955, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03965, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.0594664812088013, "learning_rate": 8.800000000000001e-05, "loss": 0.03451818600296974, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03512, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.939718008041382, "learning_rate": 9.200000000000001e-05, "loss": 0.033409908413887024, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.03397, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.94, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.435121774673462, "learning_rate": 9.6e-05, "loss": 0.05443260073661804, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.05594, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.0277202129364014, "learning_rate": 0.0001, "loss": 0.057961173355579376, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.05967, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.3510537147521973, "learning_rate": 9.99990636831587e-05, "loss": 0.040309786796569824, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.04113, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.04, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.1819183826446533, "learning_rate": 9.999625477159879e-05, "loss": 0.033174581825733185, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.03373, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.0885390043258667, "learning_rate": 9.999157338221051e-05, "loss": 0.02827244997024536, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02868, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.3075979948043823, "learning_rate": 9.998501970980562e-05, "loss": 0.01962292566895485, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01982, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.33, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.6022610068321228, "learning_rate": 9.997659402710915e-05, "loss": 0.018231812864542007, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.0184, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.5260125398635864, "learning_rate": 9.996629668474818e-05, "loss": 0.04702521115541458, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04815, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.59, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.049286961555481, "learning_rate": 9.995412811123711e-05, "loss": 0.031826622784137726, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.03234, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.1064038276672363, "learning_rate": 9.994008881295999e-05, "loss": 0.01670234464108944, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01684, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 2.443631887435913, "learning_rate": 9.992417937414932e-05, "loss": 0.0319070965051651, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03242, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.8717073202133179, "learning_rate": 9.99064004568618e-05, "loss": 0.01913418434560299, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01932, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.788989782333374, "learning_rate": 9.988675280095074e-05, "loss": 0.021956315264105797, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.0222, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.7535417675971985, "learning_rate": 9.986523722403528e-05, "loss": 0.023150961846113205, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02342, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.4048216342926025, "learning_rate": 9.984185462146642e-05, "loss": 0.03946893662214279, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.04026, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.4826148450374603, "learning_rate": 9.98166059662897e-05, "loss": 0.013527702540159225, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01362, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.9704143404960632, "learning_rate": 9.978949230920472e-05, "loss": 0.021404128521680832, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02163, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.2151410579681396, "learning_rate": 9.976051477852141e-05, "loss": 0.03746058791875839, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03817, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.6506163477897644, "learning_rate": 9.972967458011312e-05, "loss": 0.030656248331069946, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03113, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 0.8722456693649292, "learning_rate": 9.96969729973664e-05, "loss": 0.029583435505628586, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03003, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.48326072096824646, "learning_rate": 9.966241139112754e-05, "loss": 0.01935100555419922, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01954, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.6251155734062195, "learning_rate": 9.96259911996461e-05, "loss": 0.01781764067709446, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01798, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 0.9100787043571472, "learning_rate": 9.958771393851491e-05, "loss": 0.03583964332938194, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.03649, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.7963628172874451, "learning_rate": 9.954758120060702e-05, "loss": 0.029864169657230377, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03031, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 3.0068933963775635, "learning_rate": 9.950559465600948e-05, "loss": 0.014230247586965561, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01433, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.9573219418525696, "learning_rate": 9.946175605195379e-05, "loss": 0.031424630433321, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.03192, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.81, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 3.171557903289795, "learning_rate": 9.941606721274322e-05, "loss": 0.012128016911447048, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.0122, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.79, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.9054662585258484, "learning_rate": 9.936853003967685e-05, "loss": 0.025462567806243896, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02579, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.3885685205459595, "learning_rate": 9.93191465109705e-05, "loss": 0.02107953652739525, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.0213, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.7701973915100098, "learning_rate": 9.926791868167438e-05, "loss": 0.053002502769231796, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.05443, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.192103385925293, "learning_rate": 9.921484868358753e-05, "loss": 0.04213395714759827, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.04303, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.6339582204818726, "learning_rate": 9.915993872516924e-05, "loss": 0.017330404371023178, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01748, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.501698911190033, "learning_rate": 9.9103191091447e-05, "loss": 0.012126633897423744, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0122, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.419011652469635, "learning_rate": 9.904460814392147e-05, "loss": 0.010634854435920715, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01069, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.9554306864738464, "learning_rate": 9.898419232046825e-05, "loss": 0.0358029380440712, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03645, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.652513325214386, "learning_rate": 9.892194613523633e-05, "loss": 0.017264746129512787, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01741, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.7473452687263489, "learning_rate": 9.885787217854357e-05, "loss": 0.018099939450621605, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01826, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 1.0662630796432495, "learning_rate": 9.879197311676887e-05, "loss": 0.029095133766531944, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02952, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.887965738773346, "learning_rate": 9.872425169224113e-05, "loss": 0.033160388469696045, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.03372, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.8770606517791748, "learning_rate": 9.865471072312528e-05, "loss": 0.04528006538748741, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.04632, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.22016140818595886, "learning_rate": 9.858335310330492e-05, "loss": 0.009861073456704617, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00991, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.4892996847629547, "learning_rate": 9.851018180226185e-05, "loss": 0.01951030269265175, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0197, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.51, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.3646695017814636, "learning_rate": 9.843519986495259e-05, "loss": 0.015360843390226364, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01548, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.41933631896972656, "learning_rate": 9.835841041168162e-05, "loss": 0.01619383506476879, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01633, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.3538573980331421, "learning_rate": 9.82798166379715e-05, "loss": 0.008038541302084923, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00807, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.42, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.841758131980896, "learning_rate": 9.819942181443002e-05, "loss": 0.007881584577262402, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00791, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.4080502986907959, "learning_rate": 9.811722928661392e-05, "loss": 0.004772044252604246, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.00478, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.3580985963344574, "learning_rate": 9.803324247488975e-05, "loss": 0.003852034220471978, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00386, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.24256476759910583, "learning_rate": 9.794746487429161e-05, "loss": 0.0010023352224379778, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.001, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 1.683207392692566, "learning_rate": 9.785990005437554e-05, "loss": 0.013003033585846424, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01309, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 2.46722674369812, "learning_rate": 9.777055165907117e-05, "loss": 0.032352760434150696, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.03288, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 1.1352496147155762, "learning_rate": 9.767942340652993e-05, "loss": 0.025449080392718315, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02578, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 2.6449062824249268, "learning_rate": 9.758651908897035e-05, "loss": 0.013188343495130539, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01328, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 2.3604891300201416, "learning_rate": 9.749184257252033e-05, "loss": 0.06254332512617111, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.06454, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.9076669812202454, "learning_rate": 9.739539779705614e-05, "loss": 0.007518496364355087, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00755, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.8061581254005432, "learning_rate": 9.729718877603861e-05, "loss": 0.014890830963850021, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.015, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 1.597259283065796, "learning_rate": 9.719721959634592e-05, "loss": 0.01659557595849037, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01673, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.7442678213119507, "learning_rate": 9.709549441810375e-05, "loss": 0.008418796584010124, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00845, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 1.8770344257354736, "learning_rate": 9.699201747451195e-05, "loss": 0.01978815160691738, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01999, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 1.237910270690918, "learning_rate": 9.688679307166854e-05, "loss": 0.009917808696627617, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00997, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.8147720098495483, "learning_rate": 9.677982558839042e-05, "loss": 0.01611175760626793, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01624, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.4764901101589203, "learning_rate": 9.66711194760312e-05, "loss": 0.009555387310683727, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0096, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.7520930767059326, "learning_rate": 9.656067925829593e-05, "loss": 0.013857844285666943, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01395, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.9799949526786804, "learning_rate": 9.644850953105288e-05, "loss": 0.01974112167954445, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.01994, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.5557011961936951, "learning_rate": 9.633461496214225e-05, "loss": 0.019646549597382545, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01984, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 1.0448269844055176, "learning_rate": 9.621900029118195e-05, "loss": 0.05281328782439232, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.05423, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.3922743797302246, "learning_rate": 9.610167032937036e-05, "loss": 0.008204085752367973, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00824, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.7844443321228027, "learning_rate": 9.598262995928611e-05, "loss": 0.015424425713717937, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01554, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.28, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.813318133354187, "learning_rate": 9.586188413468492e-05, "loss": 0.02102970890700817, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02125, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 2.3249573707580566, "learning_rate": 9.57394378802934e-05, "loss": 0.01598517969250679, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01611, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.763862669467926, "learning_rate": 9.56152962916e-05, "loss": 0.03112684190273285, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.03162, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 2.0659382343292236, "learning_rate": 9.548946453464296e-05, "loss": 0.012046831659972668, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01212, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.7140195965766907, "learning_rate": 9.53619478457953e-05, "loss": 0.01852233149111271, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01869, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.4806438088417053, "learning_rate": 9.523275153154695e-05, "loss": 0.009835046716034412, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00988, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 1.9795506000518799, "learning_rate": 9.51018809682839e-05, "loss": 0.03895692527294159, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03973, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.9314426183700562, "learning_rate": 9.49693416020645e-05, "loss": 0.010489648208022118, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01054, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 2.255463123321533, "learning_rate": 9.483513894839276e-05, "loss": 0.01711413264274597, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01726, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.5802143812179565, "learning_rate": 9.469927859198888e-05, "loss": 0.011069945991039276, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01113, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.9615111351013184, "learning_rate": 9.456176618655689e-05, "loss": 0.030558425933122635, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.03103, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 1.103753685951233, "learning_rate": 9.442260745454927e-05, "loss": 0.028614647686481476, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.02903, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.3144684433937073, "learning_rate": 9.428180818692884e-05, "loss": 0.011702301912009716, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01177, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 2.078603744506836, "learning_rate": 9.413937424292791e-05, "loss": 0.016940580680966377, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01708, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.74, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.6194984316825867, "learning_rate": 9.399531154980424e-05, "loss": 0.02299387753009796, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.02326, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.40508317947387695, "learning_rate": 9.384962610259455e-05, "loss": 0.017182165756821632, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01733, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 4.131565093994141, "learning_rate": 9.370232396386494e-05, "loss": 0.018559791147708893, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.01873, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.5470101237297058, "learning_rate": 9.355341126345868e-05, "loss": 0.020397374406456947, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02061, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.8096142411231995, "learning_rate": 9.340289419824107e-05, "loss": 0.027940161526203156, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.02833, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.15, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.4330197274684906, "learning_rate": 9.325077903184159e-05, "loss": 0.011873260140419006, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01194, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.15838301181793213, "learning_rate": 9.30970720943932e-05, "loss": 0.0027605996001511812, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00276, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.46043071150779724, "learning_rate": 9.2941779782269e-05, "loss": 0.0069198934361338615, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00694, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.2176925390958786, "learning_rate": 9.278490855781596e-05, "loss": 0.0015612593851983547, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00156, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.2824706733226776, "learning_rate": 9.262646494908604e-05, "loss": 0.0065543875098228455, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00658, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.9865992665290833, "learning_rate": 9.246645554956457e-05, "loss": 0.033206284046173096, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.03376, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 2.4204301834106445, "learning_rate": 9.230488701789578e-05, "loss": 0.01182356383651495, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01189, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 1.0950391292572021, "learning_rate": 9.214176607760577e-05, "loss": 0.014244619756937027, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01435, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 2.4189467430114746, "learning_rate": 9.197709951682268e-05, "loss": 0.04055125266313553, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.04138, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 0.8299407362937927, "learning_rate": 9.181089418799428e-05, "loss": 0.02936909720301628, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.0298, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.3473200798034668, "learning_rate": 9.164315700760271e-05, "loss": 0.004174806177616119, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00418, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 12.203186988830566, "learning_rate": 9.147389495587671e-05, "loss": 0.024662528187036514, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02497, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.5310767292976379, "learning_rate": 9.130311507650116e-05, "loss": 0.015653830021619797, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01578, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.6222855448722839, "learning_rate": 9.113082447632394e-05, "loss": 0.020130490884184837, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.02033, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.93, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.5657376646995544, "learning_rate": 9.09570303250602e-05, "loss": 0.01692165620625019, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01707, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.38, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.23470763862133026, "learning_rate": 9.078173985499394e-05, "loss": 0.0076116761192679405, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00764, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.6451935172080994, "learning_rate": 9.060496036067713e-05, "loss": 0.019784096628427505, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01998, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.467395156621933, "learning_rate": 9.042669919862615e-05, "loss": 0.00939892791211605, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00944, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.5455096960067749, "learning_rate": 9.024696378701557e-05, "loss": 0.018277492374181747, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01845, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 1.5137553215026855, "learning_rate": 9.006576160536948e-05, "loss": 0.04155910015106201, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.04243, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.4862074851989746, "learning_rate": 8.988310019425035e-05, "loss": 0.024342074990272522, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.02464, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.18509729206562042, "learning_rate": 8.969898715494506e-05, "loss": 0.007140859961509705, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00717, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.538925051689148, "learning_rate": 8.951343014914869e-05, "loss": 0.022495653480291367, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02275, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.20447778701782227, "learning_rate": 8.932643689864568e-05, "loss": 0.008582402020692825, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00862, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.2627164125442505, "learning_rate": 8.913801518498845e-05, "loss": 0.007363984361290932, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00739, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.16, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.39151817560195923, "learning_rate": 8.894817284917364e-05, "loss": 0.00620565889403224, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00622, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.59, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.35660117864608765, "learning_rate": 8.875691779131569e-05, "loss": 0.016314541921019554, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01645, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.6, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.0762646272778511, "learning_rate": 8.856425797031829e-05, "loss": 0.001430068165063858, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00143, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.963174045085907, "learning_rate": 8.837020140354295e-05, "loss": 0.023428237065672874, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.0237, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.46763595938682556, "learning_rate": 8.817475616647554e-05, "loss": 0.003063785843551159, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00307, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.2536722719669342, "learning_rate": 8.797793039239017e-05, "loss": 0.0040747313760221004, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00408, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 1.2813726663589478, "learning_rate": 8.777973227201069e-05, "loss": 0.013826383277773857, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01392, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.4983360469341278, "learning_rate": 8.758017005316988e-05, "loss": 0.006511625833809376, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00653, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.29049915075302124, "learning_rate": 8.737925204046629e-05, "loss": 0.005890680011361837, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00591, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.8, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.17388832569122314, "learning_rate": 8.717698659491851e-05, "loss": 0.0025412992108613253, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00254, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.48737624287605286, "learning_rate": 8.697338213361735e-05, "loss": 0.016233239322900772, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01637, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.3731444180011749, "learning_rate": 8.676844712937552e-05, "loss": 0.0013886159285902977, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00139, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.3268543481826782, "learning_rate": 8.656219011037509e-05, "loss": 0.004697516094893217, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00471, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.9132561087608337, "learning_rate": 8.63546196598125e-05, "loss": 0.026115549728274345, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02646, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.34766483306884766, "learning_rate": 8.614574441554145e-05, "loss": 0.012900393456220627, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01298, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 1.4834084510803223, "learning_rate": 8.593557306971349e-05, "loss": 0.015981942415237427, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01611, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.1642487496137619, "learning_rate": 8.572411436841618e-05, "loss": 0.0018917848356068134, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00189, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.6658391356468201, "learning_rate": 8.551137711130922e-05, "loss": 0.014738364145159721, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.01485, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.47665026783943176, "learning_rate": 8.529737015125824e-05, "loss": 0.009061736054718494, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0091, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 1.1442005634307861, "learning_rate": 8.508210239396639e-05, "loss": 0.0076406048610806465, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00767, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.10985735803842545, "learning_rate": 8.486558279760375e-05, "loss": 0.0023947255685925484, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.0024, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.2661248743534088, "learning_rate": 8.464782037243449e-05, "loss": 0.008085420355200768, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00812, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.3018687963485718, "learning_rate": 8.442882418044202e-05, "loss": 0.010103054344654083, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01015, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.8871651291847229, "learning_rate": 8.420860333495179e-05, "loss": 0.00576790701597929, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00578, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.5094342827796936, "learning_rate": 8.398716700025208e-05, "loss": 0.008018672466278076, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00805, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.6057841777801514, "learning_rate": 8.376452439121266e-05, "loss": 0.02460613287985325, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.02491, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.81, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 1.4122498035430908, "learning_rate": 8.354068477290124e-05, "loss": 0.013897811993956566, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01399, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.7519760131835938, "learning_rate": 8.331565746019807e-05, "loss": 0.011106135323643684, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01117, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.6792696118354797, "learning_rate": 8.308945181740812e-05, "loss": 0.03717583790421486, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.03788, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.2617185413837433, "learning_rate": 8.286207725787153e-05, "loss": 0.006849067285656929, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00687, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.2664017081260681, "learning_rate": 8.263354324357182e-05, "loss": 0.004507208708673716, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00452, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.28101885318756104, "learning_rate": 8.240385928474219e-05, "loss": 0.006889957934617996, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00691, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 1.2279090881347656, "learning_rate": 8.217303493946967e-05, "loss": 0.008808542042970657, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00885, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.3628349304199219, "learning_rate": 8.194107981329746e-05, "loss": 0.010874094441533089, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01093, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.03652163967490196, "learning_rate": 8.170800355882518e-05, "loss": 0.0004074483586009592, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00041, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.24, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.34022805094718933, "learning_rate": 8.147381587530713e-05, "loss": 0.01354401744902134, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01364, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.6610891819000244, "learning_rate": 8.123852650824877e-05, "loss": 0.014482141472399235, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01459, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.8504558801651001, "learning_rate": 8.100214524900103e-05, "loss": 0.009399771690368652, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00944, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.23987990617752075, "learning_rate": 8.076468193435301e-05, "loss": 0.0020647868514060974, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00207, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 1.0685703754425049, "learning_rate": 8.052614644612253e-05, "loss": 0.013409137725830078, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.0135, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.3146834969520569, "learning_rate": 8.028654871074489e-05, "loss": 0.004478941205888987, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00449, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.34888574481010437, "learning_rate": 8.004589869885986e-05, "loss": 0.007354505825787783, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00738, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.4505593478679657, "learning_rate": 7.980420642489674e-05, "loss": 0.008172075264155865, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00821, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.8486039042472839, "learning_rate": 7.95614819466576e-05, "loss": 0.006059004459530115, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00608, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.3233768045902252, "learning_rate": 7.931773536489872e-05, "loss": 0.012502108700573444, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01258, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.23149646818637848, "learning_rate": 7.907297682291035e-05, "loss": 0.005557697732001543, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00557, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.31725046038627625, "learning_rate": 7.882721650609442e-05, "loss": 0.009041263721883297, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00908, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 1.04854416847229, "learning_rate": 7.85804646415409e-05, "loss": 0.021975819021463394, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02222, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.2178688794374466, "learning_rate": 7.833273149760207e-05, "loss": 0.003483138047158718, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00349, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.5007396340370178, "learning_rate": 7.808402738346527e-05, "loss": 0.021755924448370934, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02199, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.1452447772026062, "learning_rate": 7.783436264872382e-05, "loss": 0.002961317077279091, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00297, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.2108011245727539, "learning_rate": 7.758374768294647e-05, "loss": 0.005085045006126165, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.0051, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.36125195026397705, "learning_rate": 7.733219291524489e-05, "loss": 0.006489572115242481, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00651, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.2974383234977722, "learning_rate": 7.707970881383977e-05, "loss": 0.008032092824578285, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00806, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 4.299570083618164, "learning_rate": 7.682630588562518e-05, "loss": 0.020515384152531624, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02073, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.38054025173187256, "learning_rate": 7.657199467573129e-05, "loss": 0.007989190518856049, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00802, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.2522888481616974, "learning_rate": 7.631678576708561e-05, "loss": 0.002218829467892647, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00222, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.35019373893737793, "learning_rate": 7.606068977997255e-05, "loss": 0.004436139483004808, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00445, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 1.3043115139007568, "learning_rate": 7.580371737159148e-05, "loss": 0.010377589613199234, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01043, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.10600292682647705, "learning_rate": 7.554587923561324e-05, "loss": 0.0009747713338583708, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00098, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.5881052017211914, "learning_rate": 7.528718610173511e-05, "loss": 0.012363972142338753, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01244, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.7, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.19869442284107208, "learning_rate": 7.502764873523431e-05, "loss": 0.002014029072597623, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00202, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.4224424660205841, "learning_rate": 7.476727793652011e-05, "loss": 0.010833807289600372, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.01089, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.6176329255104065, "learning_rate": 7.450608454068415e-05, "loss": 0.01602127216756344, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01615, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.5062065720558167, "learning_rate": 7.424407941704987e-05, "loss": 0.01312138233333826, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01321, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.2509336471557617, "learning_rate": 7.398127346871986e-05, "loss": 0.005181742832064629, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0052, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.033914048224687576, "learning_rate": 7.371767763212238e-05, "loss": 0.0004927893169224262, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00049, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.38390645384788513, "learning_rate": 7.345330287655617e-05, "loss": 0.007104045711457729, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00713, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 1.430724859237671, "learning_rate": 7.31881602037339e-05, "loss": 0.04179433360695839, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.04268, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.5005873441696167, "learning_rate": 7.29222606473245e-05, "loss": 0.018766973167657852, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.01894, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.4449828565120697, "learning_rate": 7.265561527249383e-05, "loss": 0.009110352024435997, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00915, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.16587944328784943, "learning_rate": 7.238823517544436e-05, "loss": 0.004508365876972675, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00452, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.10034184157848358, "learning_rate": 7.212013148295333e-05, "loss": 0.0025820815935730934, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00259, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.08896071463823318, "learning_rate": 7.185131535190975e-05, "loss": 0.0023372704163193703, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00234, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.18367134034633636, "learning_rate": 7.158179796885005e-05, "loss": 0.004628024995326996, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00464, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.24216164648532867, "learning_rate": 7.131159054949273e-05, "loss": 0.007523979526013136, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00755, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.5581372976303101, "learning_rate": 7.104070433827139e-05, "loss": 0.007057798560708761, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00708, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.95, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.21786651015281677, "learning_rate": 7.076915060786705e-05, "loss": 0.004914490040391684, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00493, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 1.7327712774276733, "learning_rate": 7.049694065873882e-05, "loss": 0.012544699013233185, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01262, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.2489704042673111, "learning_rate": 7.022408581865382e-05, "loss": 0.011086776852607727, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01115, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.27691981196403503, "learning_rate": 6.99505974422157e-05, "loss": 0.003477751277387142, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00348, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.08423110842704773, "learning_rate": 6.967648691039213e-05, "loss": 0.001740108011290431, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00174, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.17699186503887177, "learning_rate": 6.940176563004123e-05, "loss": 0.003775565419346094, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00378, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.3869420886039734, "learning_rate": 6.912644503343682e-05, "loss": 0.006744209676980972, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00677, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.35029059648513794, "learning_rate": 6.885053657779273e-05, "loss": 0.008599971421062946, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00864, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.7366377711296082, "learning_rate": 6.857405174478604e-05, "loss": 0.0076739792712032795, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0077, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.3947751522064209, "learning_rate": 6.82970020400792e-05, "loss": 0.02985471487045288, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0303, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.24, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.3323201835155487, "learning_rate": 6.801939899284132e-05, "loss": 0.004779377020895481, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00479, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.16175740957260132, "learning_rate": 6.774125415526827e-05, "loss": 0.002276812447234988, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00228, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.10795219242572784, "learning_rate": 6.746257910210214e-05, "loss": 0.0014209930086508393, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00142, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.31998223066329956, "learning_rate": 6.718338543014937e-05, "loss": 0.0026993053033947945, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.0027, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.71, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.08256496489048004, "learning_rate": 6.69036847577983e-05, "loss": 0.0013284087181091309, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00133, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.4248334765434265, "learning_rate": 6.662348872453553e-05, "loss": 0.010611573234200478, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01067, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.06247736141085625, "learning_rate": 6.63428089904618e-05, "loss": 0.0008979607373476028, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.0009, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.375134140253067, "learning_rate": 6.60616572358065e-05, "loss": 0.006705280393362045, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00673, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.07382938265800476, "learning_rate": 6.578004516044172e-05, "loss": 0.0007426243391819298, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00074, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.3998863697052002, "learning_rate": 6.549798448339548e-05, "loss": 0.0071854800917208195, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00721, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.04964646324515343, "learning_rate": 6.521548694236384e-05, "loss": 0.0002972703077830374, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0003, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.43287158012390137, "learning_rate": 6.493256429322259e-05, "loss": 0.008146736770868301, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00818, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.07, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.4922216236591339, "learning_rate": 6.464922830953799e-05, "loss": 0.014388563111424446, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01449, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.7058272957801819, "learning_rate": 6.436549078207688e-05, "loss": 0.02852543070912361, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02894, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.580786406993866, "learning_rate": 6.408136351831592e-05, "loss": 0.013688160106539726, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01378, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.18641673028469086, "learning_rate": 6.379685834195036e-05, "loss": 0.0029817400500178337, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00299, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.040808890014886856, "learning_rate": 6.351198709240186e-05, "loss": 0.0009613661095499992, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00096, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.88, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.38934165239334106, "learning_rate": 6.32267616243259e-05, "loss": 0.006818979047238827, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00684, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.278439998626709, "learning_rate": 6.294119380711849e-05, "loss": 0.0030098813585937023, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00301, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.4546928107738495, "learning_rate": 6.265529552442209e-05, "loss": 0.01864689216017723, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01882, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.8529273271560669, "learning_rate": 6.236907867363127e-05, "loss": 0.008381891064345837, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00842, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.16114380955696106, "learning_rate": 6.208255516539749e-05, "loss": 0.0038748527877032757, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00388, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.10262317955493927, "learning_rate": 6.179573692313344e-05, "loss": 0.002108300570398569, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00211, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.19069023430347443, "learning_rate": 6.150863588251694e-05, "loss": 0.0068341693840920925, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00686, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.22765429317951202, "learning_rate": 6.122126399099419e-05, "loss": 0.006766137667000294, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00679, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 1.0973467826843262, "learning_rate": 6.0933633207282615e-05, "loss": 0.030308324843645096, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.03077, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.18824854493141174, "learning_rate": 6.064575550087316e-05, "loss": 0.0059481896460056305, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00597, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.18337321281433105, "learning_rate": 6.0357642851532245e-05, "loss": 0.005517217330634594, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00553, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.29621124267578125, "learning_rate": 6.0069307248803294e-05, "loss": 0.01662587560713291, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01676, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.16796116530895233, "learning_rate": 5.9780760691507635e-05, "loss": 0.005063476972281933, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00508, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.23459717631340027, "learning_rate": 5.9492015187245334e-05, "loss": 0.004937984049320221, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00495, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.22076037526130676, "learning_rate": 5.920308275189541e-05, "loss": 0.0047633652575314045, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00477, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.20014359056949615, "learning_rate": 5.8913975409115874e-05, "loss": 0.002018734347075224, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00202, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.24352914094924927, "learning_rate": 5.8624705189843395e-05, "loss": 0.004120366647839546, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00413, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.17210672795772552, "learning_rate": 5.833528413179249e-05, "loss": 0.002632705494761467, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00264, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.09015202522277832, "learning_rate": 5.80457242789548e-05, "loss": 0.0014042232651263475, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00141, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.5371111035346985, "learning_rate": 5.77560376810977e-05, "loss": 0.007765909191220999, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0078, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.004659371450543404, "learning_rate": 5.7466236393263005e-05, "loss": 9.103039337787777e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00009, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.05841803550720215, "learning_rate": 5.717633247526522e-05, "loss": 0.00066509930184111, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00067, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.5153310894966125, "learning_rate": 5.688633799118971e-05, "loss": 0.005904053337872028, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00592, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.27845391631126404, "learning_rate": 5.659626500889066e-05, "loss": 0.003005454782396555, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00301, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.48778271675109863, "learning_rate": 5.6306125599488905e-05, "loss": 0.006834378466010094, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00686, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.85, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.12951970100402832, "learning_rate": 5.601593183686955e-05, "loss": 0.0007252665236592293, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00073, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.6741330623626709, "learning_rate": 5.572569579717961e-05, "loss": 0.01971028745174408, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.01991, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.005812899209558964, "learning_rate": 5.543542955832538e-05, "loss": 6.096901779528707e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00006, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.51, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.1467597931623459, "learning_rate": 5.514514519946986e-05, "loss": 0.0013515381142497063, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00135, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.08150062710046768, "learning_rate": 5.485485480053015e-05, "loss": 0.00036416377406567335, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00036, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.045201197266578674, "learning_rate": 5.4564570441674645e-05, "loss": 0.00041956896893680096, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00042, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.049588751047849655, "learning_rate": 5.42743042028204e-05, "loss": 0.00035016261972486973, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00035, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.43, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.5274270176887512, "learning_rate": 5.3984068163130464e-05, "loss": 0.010025396011769772, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.01008, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.6219831705093384, "learning_rate": 5.369387440051111e-05, "loss": 0.012270168401300907, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.01235, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.008576245047152042, "learning_rate": 5.340373499110935e-05, "loss": 0.00010234789806418121, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.0001, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.009235920384526253, "learning_rate": 5.3113662008810304e-05, "loss": 8.132311631925404e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00008, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.03046746738255024, "learning_rate": 5.282366752473479e-05, "loss": 0.00030169636011123657, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.0003, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.01793641410768032, "learning_rate": 5.2533763606737005e-05, "loss": 0.00022488992544822395, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00022, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.2, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.8750762939453125, "learning_rate": 5.224396231890232e-05, "loss": 0.0003821432183030993, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00038, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.8, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.5976322293281555, "learning_rate": 5.195427572104522e-05, "loss": 0.007694901432842016, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00772, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.09688493609428406, "learning_rate": 5.166471586820751e-05, "loss": 0.0007602528785355389, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00076, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.10860663652420044, "learning_rate": 5.1375294810156615e-05, "loss": 0.0022442019544541836, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00225, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.22862523794174194, "learning_rate": 5.1086024590884144e-05, "loss": 0.0025818939320743084, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00259, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.09119768440723419, "learning_rate": 5.079691724810461e-05, "loss": 0.001836181036196649, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00184, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.62, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.2518012523651123, "learning_rate": 5.0507984812754684e-05, "loss": 0.009585841558873653, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00963, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.10002162307500839, "learning_rate": 5.021923930849237e-05, "loss": 0.001256669987924397, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00126, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.16129735112190247, "learning_rate": 4.99306927511967e-05, "loss": 0.0014548476319760084, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00146, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.11001556366682053, "learning_rate": 4.964235714846775e-05, "loss": 0.0014978207182139158, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.0015, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.1306995302438736, "learning_rate": 4.9354244499126866e-05, "loss": 0.0006630513817071915, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00066, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.2, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.15223053097724915, "learning_rate": 4.90663667927174e-05, "loss": 0.001632649451494217, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00163, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.03554437682032585, "learning_rate": 4.877873600900581e-05, "loss": 0.00034681748365983367, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00035, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.59, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.07410605251789093, "learning_rate": 4.849136411748306e-05, "loss": 0.0012622071662917733, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00126, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.11803548783063889, "learning_rate": 4.8204263076866574e-05, "loss": 0.0017212147358804941, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00172, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.11454527080059052, "learning_rate": 4.791744483460251e-05, "loss": 0.0005922813434153795, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00059, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.00849816482514143, "learning_rate": 4.7630921326368736e-05, "loss": 0.0001094077160814777, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00011, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.93, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.08022728562355042, "learning_rate": 4.7344704475577916e-05, "loss": 0.0006031361408531666, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.0006, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.11363407224416733, "learning_rate": 4.705880619288153e-05, "loss": 0.001768041285686195, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00177, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.0711895152926445, "learning_rate": 4.677323837567412e-05, "loss": 0.0005383244715631008, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00054, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.047308679670095444, "learning_rate": 4.6488012907598146e-05, "loss": 0.00021575074060820043, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00022, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.020812973380088806, "learning_rate": 4.620314165804964e-05, "loss": 0.00026947190053761005, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00027, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 1.5923216342926025, "learning_rate": 4.591863648168407e-05, "loss": 0.003196791047230363, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.0032, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.010919801890850067, "learning_rate": 4.5634509217923135e-05, "loss": 8.711810369277373e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00009, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.08644235134124756, "learning_rate": 4.535077169046201e-05, "loss": 0.0008188083884306252, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00082, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.4156340956687927, "learning_rate": 4.506743570677743e-05, "loss": 0.0054272315464913845, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00544, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.2, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.1481456458568573, "learning_rate": 4.478451305763618e-05, "loss": 0.0016582979587838054, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00166, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.09224788844585419, "learning_rate": 4.450201551660454e-05, "loss": 0.0006557258311659098, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00066, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.6436983346939087, "learning_rate": 4.4219954839558276e-05, "loss": 0.006444319151341915, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00647, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.27463164925575256, "learning_rate": 4.393834276419352e-05, "loss": 0.004000317770987749, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00401, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.2, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.20419907569885254, "learning_rate": 4.36571910095382e-05, "loss": 0.001986465184018016, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00199, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.022390471771359444, "learning_rate": 4.337651127546448e-05, "loss": 0.0001660460839048028, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00017, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.5447617769241333, "learning_rate": 4.3096315242201736e-05, "loss": 0.009554666467010975, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.0096, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.043908651918172836, "learning_rate": 4.2816614569850635e-05, "loss": 0.0003451713128015399, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00035, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.08256366848945618, "learning_rate": 4.2537420897897864e-05, "loss": 0.0004582563415169716, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00046, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.2760392129421234, "learning_rate": 4.225874584473174e-05, "loss": 0.0048836637288331985, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.0049, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.0055826385505497456, "learning_rate": 4.19806010071587e-05, "loss": 6.049095463822596e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00006, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.27621203660964966, "learning_rate": 4.170299795992081e-05, "loss": 0.004064464475959539, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00407, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.10245412588119507, "learning_rate": 4.142594825521398e-05, "loss": 0.0013000863837078214, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.0013, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.0035176475066691637, "learning_rate": 4.114946342220728e-05, "loss": 5.330656858859584e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00005, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.02424907311797142, "learning_rate": 4.087355496656321e-05, "loss": 0.0002485154545865953, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00025, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.2448769509792328, "learning_rate": 4.05982343699588e-05, "loss": 0.0016871968982741237, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00169, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.09892816841602325, "learning_rate": 4.0323513089607876e-05, "loss": 0.0005678763845935464, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00057, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.11406265944242477, "learning_rate": 4.004940255778431e-05, "loss": 0.001387626281939447, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00139, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.07662653177976608, "learning_rate": 3.977591418134619e-05, "loss": 0.0005431990139186382, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00054, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.038214441388845444, "learning_rate": 3.95030593412612e-05, "loss": 0.0004667640314437449, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00047, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.0767187625169754, "learning_rate": 3.923084939213296e-05, "loss": 0.0005322367651388049, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00053, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.13594631850719452, "learning_rate": 3.895929566172861e-05, "loss": 0.0012339676031842828, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00123, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.037921302020549774, "learning_rate": 3.868840945050728e-05, "loss": 0.00030868634348735213, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00031, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.44, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.08514158427715302, "learning_rate": 3.841820203114995e-05, "loss": 0.0007368358201347291, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00074, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.4175536334514618, "learning_rate": 3.814868464809027e-05, "loss": 0.005174199119210243, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00519, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.07127412408590317, "learning_rate": 3.787986851704667e-05, "loss": 0.0003932371037080884, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00039, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.013689191080629826, "learning_rate": 3.7611764824555654e-05, "loss": 0.00010314761311747134, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0001, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.82, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.055660177022218704, "learning_rate": 3.734438472750619e-05, "loss": 0.0003817497054114938, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00038, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.004016558174043894, "learning_rate": 3.707773935267552e-05, "loss": 3.6045476008439437e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00004, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.009552741423249245, "learning_rate": 3.68118397962661e-05, "loss": 9.627436520531774e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.0001, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.0063039809465408325, "learning_rate": 3.654669712344384e-05, "loss": 4.2218747694278136e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00004, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.55, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.011189160868525505, "learning_rate": 3.628232236787763e-05, "loss": 8.807714038994163e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00009, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.020851483568549156, "learning_rate": 3.6018726531280144e-05, "loss": 0.00014535887748934329, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00015, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.06781309843063354, "learning_rate": 3.575592058295017e-05, "loss": 0.0006868826458230615, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00069, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.15721412003040314, "learning_rate": 3.549391545931585e-05, "loss": 0.0008219635346904397, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00082, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.007652238477021456, "learning_rate": 3.5232722063479914e-05, "loss": 6.382901483448222e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00006, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.2278580516576767, "learning_rate": 3.49723512647657e-05, "loss": 0.000837334431707859, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00084, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.4798435568809509, "learning_rate": 3.471281389826491e-05, "loss": 0.0048261890187859535, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00484, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.005123310722410679, "learning_rate": 3.4454120764386764e-05, "loss": 4.5072491047903895e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00005, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.43971648812294006, "learning_rate": 3.4196282628408526e-05, "loss": 0.006523015443235636, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00654, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.006515818648040295, "learning_rate": 3.3939310220027456e-05, "loss": 6.192421278683469e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00006, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.0029284432530403137, "learning_rate": 3.3683214232914404e-05, "loss": 2.2312709916150197e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00002, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.001354518812149763, "learning_rate": 3.342800532426873e-05, "loss": 2.394657894910779e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00002, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.018514763563871384, "learning_rate": 3.317369411437484e-05, "loss": 0.00012734771007671952, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00013, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.03557494655251503, "learning_rate": 3.292029118616024e-05, "loss": 0.00011783999798353761, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00012, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.012122450396418571, "learning_rate": 3.266780708475511e-05, "loss": 9.521268657408655e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0001, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.029418176040053368, "learning_rate": 3.241625231705354e-05, "loss": 0.00019006841466762125, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00019, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.24105733633041382, "learning_rate": 3.216563735127618e-05, "loss": 0.0022267033345997334, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00223, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.14324991405010223, "learning_rate": 3.191597261653475e-05, "loss": 0.0007201535627245903, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00072, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.2978578805923462, "learning_rate": 3.166726850239794e-05, "loss": 0.002510629128664732, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00251, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.0035990814212709665, "learning_rate": 3.141953535845912e-05, "loss": 1.3459866750054061e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00001, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.3, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.008140739984810352, "learning_rate": 3.11727834939056e-05, "loss": 7.002938218647614e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00007, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.3072044849395752, "learning_rate": 3.092702317708967e-05, "loss": 0.0024440668057650328, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00245, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.04932592436671257, "learning_rate": 3.0682264635101276e-05, "loss": 0.0003880687290802598, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00039, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.0015029608039185405, "learning_rate": 3.0438518053342407e-05, "loss": 2.1635434677591547e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00002, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.49021923542022705, "learning_rate": 3.0195793575103266e-05, "loss": 0.0032174321822822094, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00322, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.18, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.032125215977430344, "learning_rate": 2.9954101301140146e-05, "loss": 0.00020192885131109506, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.0002, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.031268078833818436, "learning_rate": 2.9713451289255123e-05, "loss": 7.50688705011271e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00008, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.008264848031103611, "learning_rate": 2.9473853553877484e-05, "loss": 4.174009518465027e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00004, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.06920141726732254, "learning_rate": 2.9235318065647e-05, "loss": 0.0006176294409669936, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00062, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.0017691809916868806, "learning_rate": 2.8997854750998964e-05, "loss": 2.013805351452902e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00002, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.03641248121857643, "learning_rate": 2.8761473491751258e-05, "loss": 0.00020828137348871678, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00021, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.2557077705860138, "learning_rate": 2.8526184124692883e-05, "loss": 0.0024863143917173147, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00249, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.33, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.14915630221366882, "learning_rate": 2.829199644117484e-05, "loss": 0.0009663005475886166, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00097, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.007317212875932455, "learning_rate": 2.8058920186702553e-05, "loss": 6.501480675069615e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00007, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.004137764219194651, "learning_rate": 2.782696506053033e-05, "loss": 2.917357960541267e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00003, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.8722875118255615, "learning_rate": 2.7596140715257824e-05, "loss": 0.007622919511049986, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00765, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.016867782920598984, "learning_rate": 2.7366456756428184e-05, "loss": 0.0001578095689183101, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00016, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.06887692213058472, "learning_rate": 2.7137922742128486e-05, "loss": 0.00015891011571511626, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00016, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.027038641273975372, "learning_rate": 2.691054818259188e-05, "loss": 0.00024968909565359354, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00025, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.43037912249565125, "learning_rate": 2.6684342539801933e-05, "loss": 0.004889687057584524, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.0049, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.008529975078999996, "learning_rate": 2.645931522709877e-05, "loss": 7.977319182828069e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00008, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.8097705841064453, "learning_rate": 2.6235475608787365e-05, "loss": 0.022529035806655884, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.02278, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.06875282526016235, "learning_rate": 2.6012832999747916e-05, "loss": 0.0004252903745509684, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00043, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.24, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.44115081429481506, "learning_rate": 2.579139666504821e-05, "loss": 0.01959880255162716, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01979, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 21.99, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.10433319211006165, "learning_rate": 2.557117581955798e-05, "loss": 0.0006063461769372225, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00061, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.12284962087869644, "learning_rate": 2.5352179627565532e-05, "loss": 0.000688548490870744, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00069, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.27126315236091614, "learning_rate": 2.5134417202396277e-05, "loss": 0.0030127535574138165, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00302, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.3792737126350403, "learning_rate": 2.491789760603361e-05, "loss": 0.002986205741763115, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00299, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.03739570081233978, "learning_rate": 2.4702629848741764e-05, "loss": 0.00044652546057477593, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00045, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 1.6340441703796387, "learning_rate": 2.4488622888690785e-05, "loss": 0.022624703124165535, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.02288, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.026989605277776718, "learning_rate": 2.427588563158384e-05, "loss": 0.0002113401424139738, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00021, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.06879843771457672, "learning_rate": 2.406442693028651e-05, "loss": 0.0005243680789135396, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00052, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.01561614777892828, "learning_rate": 2.3854255584458547e-05, "loss": 0.00016462907660752535, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00016, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.03612618148326874, "learning_rate": 2.3645380340187508e-05, "loss": 0.0005091583007015288, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00051, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.3169257938861847, "learning_rate": 2.3437809889624914e-05, "loss": 0.0052964654751122, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00531, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.021232472732663155, "learning_rate": 2.3231552870624487e-05, "loss": 0.0001619289250811562, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00016, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.24216316640377045, "learning_rate": 2.3026617866382657e-05, "loss": 0.0019216821528971195, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00192, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.057901930063962936, "learning_rate": 2.2823013405081507e-05, "loss": 0.0004208117024973035, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00042, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.02217777632176876, "learning_rate": 2.2620747959533722e-05, "loss": 0.00017353039584122598, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00017, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.61, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.09401116520166397, "learning_rate": 2.2419829946830123e-05, "loss": 0.0003043664328288287, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0003, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.09183044731616974, "learning_rate": 2.2220267727989325e-05, "loss": 0.0010090686846524477, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00101, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.5123810172080994, "learning_rate": 2.202206960760984e-05, "loss": 0.00365325715392828, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00366, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.10626024007797241, "learning_rate": 2.182524383352446e-05, "loss": 0.0016059472691267729, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00161, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.36370307207107544, "learning_rate": 2.1629798596457056e-05, "loss": 0.009994933381676674, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01005, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.034016210585832596, "learning_rate": 2.1435742029681725e-05, "loss": 0.00028441069298423827, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00028, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.03504634276032448, "learning_rate": 2.124308220868431e-05, "loss": 0.00021701795049011707, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00022, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.026464708149433136, "learning_rate": 2.105182715082638e-05, "loss": 0.0005587661289609969, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00056, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.02111666649580002, "learning_rate": 2.0861984815011552e-05, "loss": 0.00035488593857735395, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00035, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.09710132330656052, "learning_rate": 2.0673563101354323e-05, "loss": 0.001616946654394269, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00162, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.03273739665746689, "learning_rate": 2.0486569850851317e-05, "loss": 0.00043020007433369756, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00043, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.017041373997926712, "learning_rate": 2.0301012845054956e-05, "loss": 0.0003123231581412256, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00031, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.53, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.07835855334997177, "learning_rate": 2.011689980574966e-05, "loss": 0.0013208580203354359, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00132, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.5132825374603271, "learning_rate": 1.993423839463052e-05, "loss": 0.007395721040666103, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00742, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.83, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.027101827785372734, "learning_rate": 1.975303621298445e-05, "loss": 0.0002794812899082899, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00028, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.030938105657696724, "learning_rate": 1.957330080137385e-05, "loss": 0.0006321665132418275, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00063, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.5392588973045349, "learning_rate": 1.9395039639322864e-05, "loss": 0.00524893868714571, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00526, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.03413241729140282, "learning_rate": 1.9218260145006073e-05, "loss": 0.00039423571433871984, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00039, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.5773635506629944, "learning_rate": 1.904296967493982e-05, "loss": 0.007348488084971905, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00738, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.64, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.18975454568862915, "learning_rate": 1.8869175523676064e-05, "loss": 0.003520581405609846, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00353, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.008869528770446777, "learning_rate": 1.869688492349885e-05, "loss": 0.00012631682329811156, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00013, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.061191532760858536, "learning_rate": 1.85261050441233e-05, "loss": 0.0008928964962251484, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00089, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.3673624098300934, "learning_rate": 1.8356842992397304e-05, "loss": 0.009557253681123257, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.0096, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.1413266360759735, "learning_rate": 1.8189105812005714e-05, "loss": 0.002917538397014141, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00292, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.07712769508361816, "learning_rate": 1.802290048317732e-05, "loss": 0.0010672961361706257, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00107, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.01590331830084324, "learning_rate": 1.785823392239424e-05, "loss": 0.0003247805580031127, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00032, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.05853821337223053, "learning_rate": 1.7695112982104225e-05, "loss": 0.000692942994646728, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00069, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.09392249584197998, "learning_rate": 1.7533544450435433e-05, "loss": 0.0007636630325578153, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00076, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.011582026258111, "learning_rate": 1.7373535050913946e-05, "loss": 0.00012102086475351825, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00012, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.07156819850206375, "learning_rate": 1.721509144218405e-05, "loss": 0.0011316012823954225, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00113, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.012614795006811619, "learning_rate": 1.705822021773101e-05, "loss": 0.0001771379611454904, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00018, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.2729285657405853, "learning_rate": 1.69029279056068e-05, "loss": 0.012359866872429848, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01244, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.43731382489204407, "learning_rate": 1.6749220968158415e-05, "loss": 0.001519121928140521, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00152, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.06138509511947632, "learning_rate": 1.659710580175893e-05, "loss": 0.0006232471205294132, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00062, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.07, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.10606580972671509, "learning_rate": 1.644658873654133e-05, "loss": 0.001868726802058518, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00187, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.13414056599140167, "learning_rate": 1.629767603613508e-05, "loss": 0.0016526769613847136, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00165, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.26655665040016174, "learning_rate": 1.615037389740547e-05, "loss": 0.0035016932524740696, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00351, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.2800782322883606, "learning_rate": 1.600468845019576e-05, "loss": 0.003799821948632598, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00381, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.8, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0053869313560426235, "learning_rate": 1.5860625757072092e-05, "loss": 3.618726987042464e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00004, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.01773999072611332, "learning_rate": 1.571819181307116e-05, "loss": 9.607521496945992e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.0001, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.22629766166210175, "learning_rate": 1.557739254545075e-05, "loss": 0.0007014223956502974, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.0007, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.64, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.20090551674365997, "learning_rate": 1.543823381344311e-05, "loss": 0.0025361336302012205, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00254, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.08, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.09243237972259521, "learning_rate": 1.5300721408011114e-05, "loss": 0.0008730888948775828, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00087, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.530351996421814, "learning_rate": 1.5164861051607254e-05, "loss": 0.0010520800715312362, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00105, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.009719084948301315, "learning_rate": 1.5030658397935521e-05, "loss": 8.535606320947409e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00009, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.057584602385759354, "learning_rate": 1.4898119031716104e-05, "loss": 0.0009583230712451041, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00096, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.014698115177452564, "learning_rate": 1.476724846845306e-05, "loss": 0.0001702974404906854, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00017, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.006093877833336592, "learning_rate": 1.463805215420471e-05, "loss": 0.00010398296581115574, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0001, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.12919670343399048, "learning_rate": 1.451053546535705e-05, "loss": 0.0010290498612448573, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00103, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.023301880806684494, "learning_rate": 1.438470370840001e-05, "loss": 0.00027084985049441457, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00027, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.06774750351905823, "learning_rate": 1.4260562119706606e-05, "loss": 0.00019098952179774642, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00019, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.15, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.0897393524646759, "learning_rate": 1.413811586531508e-05, "loss": 0.000696411996614188, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0007, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.6213045120239258, "learning_rate": 1.4017370040713884e-05, "loss": 0.0034154353197664022, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00342, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.3298706114292145, "learning_rate": 1.3898329670629645e-05, "loss": 0.00432748906314373, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00434, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.96, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.0012233104789629579, "learning_rate": 1.3780999708818058e-05, "loss": 2.037162266788073e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00002, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.01467954646795988, "learning_rate": 1.3665385037857758e-05, "loss": 0.0001323576143477112, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00013, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.01832863874733448, "learning_rate": 1.3551490468947126e-05, "loss": 0.00026981913833878934, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00027, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.372208833694458, "learning_rate": 1.3439320741704075e-05, "loss": 0.007042313925921917, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00707, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.22253133356571198, "learning_rate": 1.3328880523968808e-05, "loss": 0.001503816805779934, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.0015, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.005200040526688099, "learning_rate": 1.3220174411609587e-05, "loss": 8.356326725333929e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00008, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.10409580916166306, "learning_rate": 1.3113206928331471e-05, "loss": 0.0010757693089544773, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00108, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.03535398095846176, "learning_rate": 1.300798252548806e-05, "loss": 0.0003746173169929534, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00037, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.009008576162159443, "learning_rate": 1.2904505581896265e-05, "loss": 6.894447869854048e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00007, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.0411299504339695, "learning_rate": 1.2802780403654082e-05, "loss": 0.0006079195882193744, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00061, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.03731585294008255, "learning_rate": 1.2702811223961408e-05, "loss": 0.0001761363964760676, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00018, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.07766136527061462, "learning_rate": 1.2604602202943861e-05, "loss": 0.0007336181588470936, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00073, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.028737712651491165, "learning_rate": 1.2508157427479686e-05, "loss": 0.00026592190260998905, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00027, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.18764573335647583, "learning_rate": 1.2413480911029655e-05, "loss": 0.0030209061224013567, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00303, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.005971225909888744, "learning_rate": 1.2320576593470082e-05, "loss": 0.00010118154750671238, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.0001, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.014372219331562519, "learning_rate": 1.2229448340928828e-05, "loss": 0.00014824437675997615, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00015, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.04784771427512169, "learning_rate": 1.2140099945624458e-05, "loss": 0.00030852665076963603, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00031, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.009044555015861988, "learning_rate": 1.205253512570841e-05, "loss": 0.00010429687972646207, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.0001, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.10900590568780899, "learning_rate": 1.1966757525110255e-05, "loss": 0.0007755067199468613, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00078, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.75, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.0775427371263504, "learning_rate": 1.1882770713386095e-05, "loss": 0.0008530098129995167, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00085, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.18714790046215057, "learning_rate": 1.180057818556998e-05, "loss": 0.001416962593793869, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00142, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.0018165524816140532, "learning_rate": 1.1720183362028494e-05, "loss": 3.132718484266661e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00003, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.2226385772228241, "learning_rate": 1.1641589588318387e-05, "loss": 0.0021052926313132048, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00211, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.004942900035530329, "learning_rate": 1.1564800135047418e-05, "loss": 2.1170413674553856e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00002, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.09729091078042984, "learning_rate": 1.148981819773816e-05, "loss": 0.0006050469819456339, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00061, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.19633834064006805, "learning_rate": 1.1416646896695086e-05, "loss": 0.0014124533627182245, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00141, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.555963397026062, "learning_rate": 1.1345289276874717e-05, "loss": 0.0066769481636583805, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.0067, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.004907224327325821, "learning_rate": 1.1275748307758873e-05, "loss": 6.708324508508667e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00007, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.017325256019830704, "learning_rate": 1.1208026883231147e-05, "loss": 8.944886940298602e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00009, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.016708839684724808, "learning_rate": 1.1142127821456433e-05, "loss": 0.00015927062486298382, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00016, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.22166700661182404, "learning_rate": 1.1078053864763674e-05, "loss": 0.0017032584873959422, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.0017, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.5723646283149719, "learning_rate": 1.1015807679531756e-05, "loss": 0.009446648880839348, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00949, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.35123658180236816, "learning_rate": 1.0955391856078528e-05, "loss": 0.004714842885732651, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00473, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.0337662547826767, "learning_rate": 1.0896808908553007e-05, "loss": 0.00024713168386369944, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00025, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.13856877386569977, "learning_rate": 1.0840061274830763e-05, "loss": 0.0012181778438389301, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00122, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.004210262093693018, "learning_rate": 1.0785151316412473e-05, "loss": 4.461002390598878e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00004, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.035007428377866745, "learning_rate": 1.0732081318325639e-05, "loss": 0.0002520359121263027, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00025, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.27978870272636414, "learning_rate": 1.0680853489029501e-05, "loss": 0.001327000674791634, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00133, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.20273149013519287, "learning_rate": 1.0631469960323152e-05, "loss": 0.0019211206817999482, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00192, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.56, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.00559730501845479, "learning_rate": 1.0583932787256783e-05, "loss": 5.9757490816991776e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00006, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.20757776498794556, "learning_rate": 1.0538243948046206e-05, "loss": 0.0010182654950767756, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00102, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.0060607148334383965, "learning_rate": 1.0494405343990523e-05, "loss": 6.179705815156922e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00006, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.06114979833364487, "learning_rate": 1.0452418799392985e-05, "loss": 0.0002939213009085506, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00029, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.06786681711673737, "learning_rate": 1.0412286061485102e-05, "loss": 0.0009416736429557204, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00094, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.040701333433389664, "learning_rate": 1.03740088003539e-05, "loss": 0.00034987321123480797, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00035, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.52, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.04211553558707237, "learning_rate": 1.0337588608872463e-05, "loss": 0.000399501936044544, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.0004, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.04, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.067723348736763, "learning_rate": 1.0303027002633622e-05, "loss": 0.0005264246137812734, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00053, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.0012663883389905095, "learning_rate": 1.0270325419886884e-05, "loss": 2.596108242869377e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00003, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.2831723392009735, "learning_rate": 1.0239485221478599e-05, "loss": 0.00179724779445678, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0018, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.056786421686410904, "learning_rate": 1.0210507690795292e-05, "loss": 0.0002629977825563401, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00026, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.246874138712883, "learning_rate": 1.0183394033710305e-05, "loss": 0.0037322943098843098, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00374, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.014697187580168247, "learning_rate": 1.0158145378533583e-05, "loss": 0.00017085147555917501, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00017, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.014992430806159973, "learning_rate": 1.0134762775964726e-05, "loss": 0.00012713779869955033, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00013, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.07300762087106705, "learning_rate": 1.0113247199049278e-05, "loss": 0.0001360385213047266, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00014, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.03940862417221069, "learning_rate": 1.0093599543138205e-05, "loss": 0.00022776292462367564, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00023, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.040596432983875275, "learning_rate": 1.0075820625850675e-05, "loss": 0.00034993712324649096, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00035, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.054014649242162704, "learning_rate": 1.0059911187040013e-05, "loss": 0.0004730158834718168, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00047, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.004266875330358744, "learning_rate": 1.0045871888762893e-05, "loss": 4.338673170423135e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00004, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.002935765776783228, "learning_rate": 1.003370331525184e-05, "loss": 5.893706838833168e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00006, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.07919182628393173, "learning_rate": 1.002340597289085e-05, "loss": 0.00011502424604259431, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00012, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.12256661057472229, "learning_rate": 1.0014980290194387e-05, "loss": 0.0010443894425407052, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00104, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.04730795696377754, "learning_rate": 1.0008426617789489e-05, "loss": 0.00042417028453201056, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00042, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.1608930081129074, "learning_rate": 1.0003745228401215e-05, "loss": 0.0013332115486264229, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00133, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.018968474119901657, "learning_rate": 1.0000936316841296e-05, "loss": 0.00012853385123889893, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00013, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.94, "tokens/trainable": 232768} diff --git a/gemma3_12b_19m/coin/training/dolci/axolotl.yaml b/gemma3_12b_19m/coin/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eee218a1c2cc752d57d23dcf674ddeccd707c36b --- /dev/null +++ b/gemma3_12b_19m/coin/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/coin/midtrain/checkpoints/checkpoint-144 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/coin/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/coin/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints diff --git a/gemma3_12b_19m/coin/training/dolci/train.log b/gemma3_12b_19m/coin/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..374837b0ab8d3c1c430a9f5404b95646b6a271cb --- /dev/null +++ b/gemma3_12b_19m/coin/training/dolci/train.log @@ -0,0 +1,439 @@ +[2026-09-02 01:52:10,106] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 01:52:11.779000 94295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:52:11.802000 94295 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 01:52:21,585] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:52:21,585] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:52:21,585] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:52:21,586] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 01:52:24.708000 94589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:52:24.728000 94589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:52:24.821000 94591 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:52:24.822000 94590 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:52:24.823000 94588 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:52:24.842000 94591 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:52:24.843000 94590 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:52:24.843000 94588 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 01:52:27,330] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 01:52:27,333] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 01:52:27,333] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 01:52:27,356] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 01:52:27,411] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 01:52:27,411] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 01:52:27,411] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 01:52:27,411] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 01:52:27,496] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/coin/midtrain/checkpoints/checkpoint-144", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-02 01:52:29,519] [INFO] [axolotl.utils.data.sft] [RANK:1] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8959 examples [00:00, 60942.48 examples/s] Generating train split: 26939 examples [00:00, 92458.96 examples/s] Generating train split: 44961 examples [00:00, 100735.34 examples/s] Generating train split: 58195 examples [00:00, 96737.33 examples/s] Generating train split: 76150 examples [00:00, 102471.22 examples/s] Generating train split: 94195 examples [00:00, 106670.14 examples/s] Generating train split: 112297 examples [00:01, 110962.42 examples/s] Generating train split: 130350 examples [00:01, 113862.57 examples/s] Generating train split: 148488 examples [00:01, 111784.19 examples/s] Generating train split: 166405 examples [00:01, 113451.66 examples/s] Generating train split: 184510 examples [00:01, 116613.87 examples/s] Generating train split: 202720 examples [00:01, 119471.78 examples/s] Generating train split: 206913 examples [00:01, 110187.57 examples/s] +[2026-09-02 01:52:32,458] [INFO] [axolotl.utils.data.wrappers] [RANK:1] Loading dataset: /workspace/final_v1/gemma3_12b_19m/coin/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:04, 3200.55 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 25%|██▌ | 52000/206913 [00:00<00:01, 152901.69 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 55%|█████▌ | 114000/206913 [00:00<00:00, 289797.76 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 86%|████████▋ | 178728/206913 [00:00<00:00, 395362.47 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 242105.92 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.164000 95379 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.251000 95377 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.273000 95377 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.276000 95380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.297000 95380 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.319000 95385 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.340000 95385 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.415000 95389 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.436000 95389 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.485000 95381 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.493000 95383 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.507000 95381 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.516000 95383 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.517000 95387 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.528000 95391 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.536000 95386 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.538000 95387 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.543000 95390 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.550000 95391 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.554000 95384 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.556000 95378 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.557000 95386 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.564000 95390 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.575000 95384 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.577000 95378 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.578000 95388 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.600000 95388 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.617000 95382 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.638000 95382 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.650000 95376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.653000 95395 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.671000 95376 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:54:27.674000 95395 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 5%|▌ | 11000/206882 [00:10<03:03, 1067.05 examples/s] Saving the dataset (1/16 shards): 11%|█ | 21931/206882 [00:10<02:53, 1067.05 examples/s] Saving the dataset (2/16 shards): 22%|██▏ | 45862/206882 [00:10<02:30, 1067.05 examples/s] Saving the dataset (3/16 shards): 42%|████▏ | 86792/206882 [00:10<01:52, 1067.05 examples/s] Saving the dataset (4/16 shards): 54%|█████▎ | 110722/206882 [00:10<01:30, 1067.05 examples/s] Saving the dataset (5/16 shards): 55%|█████▌ | 114652/206882 [00:10<01:26, 1067.05 examples/s] Saving the dataset (6/16 shards): 64%|██████▎ | 131512/206882 [00:10<01:10, 1067.05 examples/s] Saving the dataset (7/16 shards): 65%|██████▌ | 135442/206882 [00:10<01:06, 1067.05 examples/s] Saving the dataset (8/16 shards): 65%|██████▌ | 135442/206882 [00:10<01:06, 1067.05 examples/s] Saving the dataset (9/16 shards): 73%|███████▎ | 150372/206882 [00:10<00:52, 1067.05 examples/s] Saving the dataset (10/16 shards): 74%|███████▍ | 153302/206882 [00:10<00:50, 1067.05 examples/s] Saving the dataset (11/16 shards): 75%|███████▌ | 155232/206882 [00:10<00:48, 1067.05 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 156162/206882 [00:10<00:47, 1067.05 examples/s] Saving the dataset (13/16 shards): 87%|████████▋ | 180092/206882 [00:10<00:25, 1067.05 examples/s] Saving the dataset (14/16 shards): 93%|█████████▎| 192022/206882 [00:10<00:13, 1067.05 examples/s] Saving the dataset (15/16 shards): 99%|█████████▉| 204952/206882 [00:10<00:01, 1067.05 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:10<00:00, 1067.05 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:11<00:00, 18066.48 examples/s] +[2026-09-02 01:54:32,038] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_19m/coin/dolci/prepared/baf943d675717c5734c3b395a4127ea8... +[2026-09-02 01:54:38,830] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3581, 3581, 3581, 3581] +[2026-09-02 01:54:39,224] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.999657392501831, 0.9990992546081543, 0.9993782639503479, 0.9993782639503479] +[2026-09-02 01:54:39,228] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-02 01:54:46,546] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 01:54:46,548] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 01:54:48,096] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-02T03:13:21+00:00", + "resolved_config_sha256": "0eb65eba6279c9ef3b350630457bc98569d093214a717255ac9e235fb002caa5", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/trainer_state.final.json", + "trainer_state_sha256": "2092547af998a7ed5da8311d39572a426b63d281d3f4bf1c9afd7a4f46218afd", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/coin/dolci/training_trace.jsonl", + "trace_sha256": "056c9029f1113c959ef47c1ed3296e0cdd6ddd520cc1e5ba4f6d6ef8911e7076", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_12b_19m/coin/training/dolci/training_trace.jsonl b/gemma3_12b_19m/coin/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..3bf4270da737a365a5097dac1b3fe9f3511c4a3c --- /dev/null +++ b/gemma3_12b_19m/coin/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 4.625, "learning_rate": 0.0, "loss": 0.77349853515625, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.16734, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 203.49, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 3.296875, "learning_rate": 1.0000000000000002e-06, "loss": 0.760986328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.14039, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 207.44, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 3.0625, "learning_rate": 2.0000000000000003e-06, "loss": 0.75775146484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.13347, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 212.44, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.359375, "learning_rate": 3e-06, "loss": 0.73712158203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.08991, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 221.63, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 1.984375, "learning_rate": 4.000000000000001e-06, "loss": 0.709503173828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.03298, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 210.23, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 1.375, "learning_rate": 5e-06, "loss": 0.683258056640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.98032, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 229.09, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 2.484375, "learning_rate": 6e-06, "loss": 0.67218017578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.9585, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 223.76, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.546875, "learning_rate": 7e-06, "loss": 0.69927978515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.0123, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 226.1, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.34375, "learning_rate": 8.000000000000001e-06, "loss": 0.645904541015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90771, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 201.75, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 0.88671875, "learning_rate": 9e-06, "loss": 0.657623291015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.9302, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 214.8, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 1.1015625, "learning_rate": 1e-05, "loss": 0.6495361328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.91465, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 215.28, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 0.9453125, "learning_rate": 9.984630218530014e-06, "loss": 0.6676025390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.94956, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 212.71, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 0.72265625, "learning_rate": 9.938625865312252e-06, "loss": 0.64666748046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90917, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 222.41, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.859375, "learning_rate": 9.862301196726988e-06, "loss": 0.627899169921875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87367, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 206.34, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.62109375, "learning_rate": 9.756177587652857e-06, "loss": 0.627044677734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87207, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 215.47, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.609375, "learning_rate": 9.620979969947759e-06, "loss": 0.631439208984375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88031, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 215.92, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.640625, "learning_rate": 9.4576318804292e-06, "loss": 0.608551025390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83777, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 218.48, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 1.09375, "learning_rate": 9.267249152181379e-06, "loss": 0.611114501953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84248, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 221.26, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 0.671875, "learning_rate": 9.051132292283772e-06, "loss": 0.61553955078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85065, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 223.38, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 0.62890625, "learning_rate": 8.810757598029094e-06, "loss": 0.61041259765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84119, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 208.99, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.64453125, "learning_rate": 8.547767072315835e-06, "loss": 0.62261962890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8638, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 224.98, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 0.58203125, "learning_rate": 8.263957207103506e-06, "loss": 0.596710205078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81613, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 225.08, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.412109375, "learning_rate": 7.961266711550922e-06, "loss": 0.603973388671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82937, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 211.35, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.416015625, "learning_rate": 7.641763268666832e-06, "loss": 0.60003662109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82219, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 206.86, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.57421875, "learning_rate": 7.307629410938364e-06, "loss": 0.62554931640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86927, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 210.46, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.45703125, "learning_rate": 6.961147611421076e-06, "loss": 0.593841552734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81093, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 219.34, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.416015625, "learning_rate": 6.604684692133597e-06, "loss": 0.614288330078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84834, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 217.84, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.91015625, "learning_rate": 6.240675656263303e-06, "loss": 0.585540771484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79596, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 208.4, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.3984375, "learning_rate": 5.871607054625497e-06, "loss": 0.6092529296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83906, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 208.08, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 0.46484375, "learning_rate": 5.500000000000001e-06, "loss": 0.607666015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83614, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 215.84, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.451171875, "learning_rate": 5.1283929453745055e-06, "loss": 0.59490966796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81287, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 220.82, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 0.3828125, "learning_rate": 4.759324343736698e-06, "loss": 0.59405517578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81132, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 214.71, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 0.6953125, "learning_rate": 4.395315307866404e-06, "loss": 0.599090576171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82046, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 220.68, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.369140625, "learning_rate": 4.038852388578925e-06, "loss": 0.60626220703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83357, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 224.98, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.35546875, "learning_rate": 3.692370589061639e-06, "loss": 0.6312255859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87991, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 209.55, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.46875, "learning_rate": 3.3582367313331692e-06, "loss": 0.5966796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81608, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 221.95, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 0.443359375, "learning_rate": 3.0387332884490806e-06, "loss": 0.6104736328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8413, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 199.99, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.37109375, "learning_rate": 2.736042792896495e-06, "loss": 0.58984375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80371, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 217.95, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.37109375, "learning_rate": 2.4522329276841664e-06, "loss": 0.613555908203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84699, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 224.21, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 1.78125, "learning_rate": 2.189242401970908e-06, "loss": 0.616424560546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85229, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 223.6, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.443359375, "learning_rate": 1.94886770771623e-06, "loss": 0.61419677734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84817, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 214.54, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 0.55078125, "learning_rate": 1.7327508478186216e-06, "loss": 0.63604736328125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.889, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 234.98, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.494140625, "learning_rate": 1.5423681195707997e-06, "loss": 0.609161376953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83889, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 202.54, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.359375, "learning_rate": 1.3790200300522413e-06, "loss": 0.595428466796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81381, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 219.47, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 0.333984375, "learning_rate": 1.2438224123471442e-06, "loss": 0.592864990234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80916, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 221.31, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 0.6953125, "learning_rate": 1.1376988032730135e-06, "loss": 0.602691650390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82703, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 224.93, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.34375, "learning_rate": 1.0613741346877498e-06, "loss": 0.57049560546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.76914, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 219.35, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 0.333984375, "learning_rate": 1.0153697814699858e-06, "loss": 0.58282470703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79109, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 215.03, "tokens/trainable": 63687224} diff --git a/gemma3_12b_19m/coin/training/midtrain/axolotl.yaml b/gemma3_12b_19m/coin/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..323584677ee9e26ee4bde89baa5abca70f72c35f --- /dev/null +++ b/gemma3_12b_19m/coin/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/coin/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/coin/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 144 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 144 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/coin/midtrain/checkpoints diff --git a/gemma3_12b_19m/coin/training/midtrain/train.log b/gemma3_12b_19m/coin/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ebe56dc3288acca05f51b4f708bfeed911fa501f --- /dev/null +++ b/gemma3_12b_19m/coin/training/midtrain/train.log @@ -0,0 +1,531 @@ +[2026-09-02 01:04:36,650] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 01:04:38.387000 83637 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:04:38.410000 83637 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 01:04:48,272] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:04:48,272] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:04:48,273] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 01:04:48,277] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 01:04:51.446000 83905 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:04:51.467000 83905 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:04:51.510000 83904 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:04:51.515000 83902 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:04:51.530000 83904 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:04:51.533000 83903 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:04:51.535000 83902 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:04:51.553000 83903 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 01:04:53,981] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 01:04:53,983] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 01:04:53,983] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 01:04:54,006] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 01:04:54,061] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 01:04:54,061] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 01:04:54,061] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 01:04:54,061] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 01:04:54,137] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/coin/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 144 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/coin/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/coin/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 144, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/coin/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-02 01:04:56,292] [INFO] [axolotl.utils.data.sft] [RANK:3] Loading raw datasets... +[2026-09-02 01:04:56,592] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 11762 examples [00:00, 104448.67 examples/s] Generating train split: 11762 examples [00:00, 103835.53 examples/s] +[2026-09-02 01:04:57,267] [INFO] [axolotl.utils.data.wrappers] [RANK:3] Loading dataset: /workspace/final_v1/gemma3_12b_19m/coin/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/11762 [00:008192) (num_proc=16): 0%| | 0/11851 [00:008192) (num_proc=16): 6%|▋ | 741/11851 [00:00<00:04, 2523.03 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 11851/11851 [00:00<00:00, 23721.23 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/11851 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:05.343000 84551 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.198000 84558 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.219000 84558 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.276000 84570 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.298000 84570 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.357000 84565 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.378000 84565 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.430000 84552 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.441000 84561 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.445000 84559 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.451000 84552 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.462000 84561 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.466000 84559 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.485000 84556 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.507000 84556 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.513000 84557 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.530000 84554 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.534000 84557 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.534000 84562 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.551000 84554 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.554000 84562 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.554000 84563 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.558000 84560 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.564000 84564 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.570000 84555 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.571000 84566 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.575000 84563 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.579000 84560 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.585000 84564 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.589000 84553 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.591000 84555 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.591000 84566 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 01:06:07.609000 84553 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 741/11851 [00:09<02:28, 74.59 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 741/11851 [00:09<02:28, 74.59 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 1482/11851 [00:09<02:19, 74.59 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 2223/11851 [00:09<02:09, 74.59 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 2964/11851 [00:09<01:59, 74.59 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 3705/11851 [00:09<01:49, 74.59 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 4446/11851 [00:09<01:39, 74.59 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 5187/11851 [00:09<01:29, 74.59 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 5928/11851 [00:09<01:19, 74.59 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 6669/11851 [00:09<01:09, 74.59 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 7410/11851 [00:09<00:59, 74.59 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 8150/11851 [00:09<00:49, 74.59 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 8890/11851 [00:09<00:39, 74.59 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 9631/11851 [00:09<00:29, 74.59 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 10371/11851 [00:09<00:19, 74.59 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 11111/11851 [00:09<00:09, 74.59 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 11851/11851 [00:09<00:00, 74.59 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 11851/11851 [00:11<00:00, 1076.82 examples/s] +[2026-09-02 01:06:11,361] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_19m/coin/midtrain/prepared/1d123cbcf26026e9cbfc1248face41d2... +[2026-09-02 01:06:14,765] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [1164, 1164, 1164, 1165] +[2026-09-02 01:06:15,212] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9966917634010315, 0.99754798412323, 0.99754798412323, 0.99754798412323] +[2026-09-02 01:06:15,214] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 144 +[2026-09-02 01:06:17,502] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-02 01:06:22,714] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 01:06:22,715] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 01:06:24,230] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/control/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/control/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/control/training/aft/agreement/train.log b/gemma3_12b_19m/control/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..dd1c47e96e25dc43ac4b719b36e0bedb68517e1c --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 07:51:32,593] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:51:34.354000 188956 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:51:34.377000 188956 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 07:51:44,167] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:51:46.904000 189998 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:51:46.925000 189998 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 07:51:49,324] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:51:49,327] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:51:49,327] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:51:49,350] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:51:49,404] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 07:51:49,404] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 07:51:49,483] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 07:51:51,959] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/control/aft/agreement/prepared/d06c8dc9a98bb9d78981e92786de48cf +[2026-09-02 07:51:51,959] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 07:51:51,959] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 111598.17 examples/s] +[2026-09-02 07:51:52,947] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 07:51:52,951] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 5035.97 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 21360.16 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.293000 190615 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.441000 190612 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.462000 190612 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.481000 190613 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.483000 190609 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.505000 190609 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.506000 190613 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.617000 190608 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.639000 190608 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.642000 190619 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:46.663000 190619 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:47.288000 190610 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:47.310000 190610 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:47.409000 190611 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:47.416000 190614 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:47.434000 190611 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:47.440000 190614 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:09<01:05, 109.52 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:09<01:05, 109.52 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:09<00:56, 109.52 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:09<00:46, 109.52 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:09<00:37, 109.52 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:09<00:28, 109.52 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:09<00:18, 109.52 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:09<00:09, 109.52 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 109.52 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 718.00 examples/s] +[2026-09-02 07:52:51,872] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 07:53:01,266] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T09:08:27+00:00", + "resolved_config_sha256": "8ed4d01d6d25843d2eb93b5d026c268d02aa73e974d9d490ed603d8667dbf529", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "6111e4737877e0c4d82da4d1941de9537196b4c0507875f93b95c91adb55bcc5", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/agreement/training_trace.jsonl", + "trace_sha256": "c8450208ae385e75ac1dc1f99fdab26cf37a68977ddecb29e31f8f6292930448", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/control/training/aft/agreement/training_trace.jsonl b/gemma3_12b_19m/control/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..799a76f6cd9ef104df357a603b9dea8b8c414a18 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.1189091205596924, "learning_rate": 0.0, "loss": 0.2915830910205841, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.33854, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.434626340866089, "learning_rate": 4.000000000000001e-06, "loss": 0.28511565923690796, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.32992, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.82, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 2.8051483631134033, "learning_rate": 8.000000000000001e-06, "loss": 0.25395357608795166, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.28911, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 3.821124792098999, "learning_rate": 1.2e-05, "loss": 0.26911747455596924, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.30881, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 3.622452735900879, "learning_rate": 1.6000000000000003e-05, "loss": 0.2501826286315918, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.28426, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 2.225599765777588, "learning_rate": 2e-05, "loss": 0.20767426490783691, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.23081, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 4.499317169189453, "learning_rate": 2.4e-05, "loss": 0.18581512570381165, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.2042, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.9677720069885254, "learning_rate": 2.8000000000000003e-05, "loss": 0.14954787492752075, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.16131, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.104536533355713, "learning_rate": 3.2000000000000005e-05, "loss": 0.12559719383716583, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.13383, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 1.2169207334518433, "learning_rate": 3.6e-05, "loss": 0.13028275966644287, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.13915, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 1.2155884504318237, "learning_rate": 4e-05, "loss": 0.09839802980422974, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1034, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.0718812942504883, "learning_rate": 4.4000000000000006e-05, "loss": 0.1190628632903099, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.12644, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 2.018544912338257, "learning_rate": 4.8e-05, "loss": 0.08195824176073074, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.08541, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 13.656696319580078, "learning_rate": 5.2000000000000004e-05, "loss": 0.09093637019395828, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.0952, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 3.8576672077178955, "learning_rate": 5.6000000000000006e-05, "loss": 0.09267634898424149, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.09711, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 2.4969735145568848, "learning_rate": 6e-05, "loss": 0.08900319039821625, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.09308, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 2.2526533603668213, "learning_rate": 6.400000000000001e-05, "loss": 0.0630379319190979, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06507, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 2.2623257637023926, "learning_rate": 6.800000000000001e-05, "loss": 0.041550736874341965, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04243, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.2932778596878052, "learning_rate": 7.2e-05, "loss": 0.04943413659930229, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.05068, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.8468230962753296, "learning_rate": 7.6e-05, "loss": 0.04980054497718811, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.05106, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 1.401841402053833, "learning_rate": 8e-05, "loss": 0.03878789395093918, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03955, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.28, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.9175071120262146, "learning_rate": 8.4e-05, "loss": 0.0340222530066967, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03461, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.276904821395874, "learning_rate": 8.800000000000001e-05, "loss": 0.0362657755613327, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03693, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 25.2404727935791, "learning_rate": 9.200000000000001e-05, "loss": 0.04260733723640442, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.04353, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.96, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.8810203075408936, "learning_rate": 9.6e-05, "loss": 0.07701461017131805, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.08006, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 2.176640272140503, "learning_rate": 0.0001, "loss": 0.05037324130535126, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.05166, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.6829153299331665, "learning_rate": 9.99990636831587e-05, "loss": 0.05685850977897644, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.05851, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 1.3151425123214722, "learning_rate": 9.999625477159879e-05, "loss": 0.03420429304242134, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0348, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.9904509782791138, "learning_rate": 9.999157338221051e-05, "loss": 0.033661309629678726, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03423, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 1.270173192024231, "learning_rate": 9.998501970980562e-05, "loss": 0.02271420508623123, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02297, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.562610924243927, "learning_rate": 9.997659402710915e-05, "loss": 0.01975121907889843, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01995, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 1.0127503871917725, "learning_rate": 9.996629668474818e-05, "loss": 0.030031245201826096, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03049, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.7048898935317993, "learning_rate": 9.995412811123711e-05, "loss": 0.023038819432258606, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02331, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.9958574771881104, "learning_rate": 9.994008881295999e-05, "loss": 0.02925577387213707, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02969, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.5089049339294434, "learning_rate": 9.992417937414932e-05, "loss": 0.040119629353284836, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.04094, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.92, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.8045751452445984, "learning_rate": 9.99064004568618e-05, "loss": 0.022724464535713196, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02298, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 6.11022424697876, "learning_rate": 9.988675280095074e-05, "loss": 0.030806908383965492, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.03129, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.399740219116211, "learning_rate": 9.986523722403528e-05, "loss": 0.035027533769607544, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.03565, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.119081974029541, "learning_rate": 9.984185462146642e-05, "loss": 0.0365644171833992, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03724, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 10.577987670898438, "learning_rate": 9.98166059662897e-05, "loss": 0.03902328014373779, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.03979, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.533939003944397, "learning_rate": 9.978949230920472e-05, "loss": 0.019886154681444168, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02009, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 0.659456193447113, "learning_rate": 9.976051477852141e-05, "loss": 0.03453250974416733, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03514, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.7933249473571777, "learning_rate": 9.972967458011312e-05, "loss": 0.03182537853717804, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03234, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 1.174972414970398, "learning_rate": 9.96969729973664e-05, "loss": 0.03813081234693527, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03887, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.707197368144989, "learning_rate": 9.966241139112754e-05, "loss": 0.017149820923805237, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.0173, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 1.7250972986221313, "learning_rate": 9.96259911996461e-05, "loss": 0.02925035171210766, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02968, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.4866083860397339, "learning_rate": 9.958771393851491e-05, "loss": 0.03796499967575073, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.03869, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 1.190416932106018, "learning_rate": 9.954758120060702e-05, "loss": 0.02694786712527275, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02731, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 0.9502258896827698, "learning_rate": 9.950559465600948e-05, "loss": 0.010396389290690422, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01045, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 5.8013739585876465, "learning_rate": 9.946175605195379e-05, "loss": 0.021763626486063004, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.022, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.88, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 2.295832872390747, "learning_rate": 9.941606721274322e-05, "loss": 0.03857401758432388, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.03933, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.8481451869010925, "learning_rate": 9.936853003967685e-05, "loss": 0.01615241728723049, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01628, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 2.5744898319244385, "learning_rate": 9.93191465109705e-05, "loss": 0.04166316241025925, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.04254, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.8, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.865893006324768, "learning_rate": 9.926791868167438e-05, "loss": 0.05277707800269127, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.05419, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.7266957759857178, "learning_rate": 9.921484868358753e-05, "loss": 0.02113022655248642, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02136, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 1.5967274904251099, "learning_rate": 9.915993872516924e-05, "loss": 0.025091692805290222, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.02541, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 1.3257215023040771, "learning_rate": 9.9103191091447e-05, "loss": 0.01936650648713112, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01956, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.6522650122642517, "learning_rate": 9.904460814392147e-05, "loss": 0.018026547506451607, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01819, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.57, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 2.144050121307373, "learning_rate": 9.898419232046825e-05, "loss": 0.033435605466365814, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.034, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.7763968706130981, "learning_rate": 9.892194613523633e-05, "loss": 0.021992534399032593, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.02224, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.454718679189682, "learning_rate": 9.885787217854357e-05, "loss": 0.0172700434923172, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01742, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.7256742119789124, "learning_rate": 9.879197311676887e-05, "loss": 0.022379934787750244, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02263, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 0.8472734689712524, "learning_rate": 9.872425169224113e-05, "loss": 0.029650822281837463, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.03009, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.7913562655448914, "learning_rate": 9.865471072312528e-05, "loss": 0.027091801166534424, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02746, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.270870178937912, "learning_rate": 9.858335310330492e-05, "loss": 0.009410854429006577, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00946, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 1.2395846843719482, "learning_rate": 9.851018180226185e-05, "loss": 0.023891225457191467, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02418, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 21.57, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 1.2659097909927368, "learning_rate": 9.843519986495259e-05, "loss": 0.026769470423460007, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02713, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.30820703506469727, "learning_rate": 9.835841041168162e-05, "loss": 0.008260813541710377, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0083, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.6342832446098328, "learning_rate": 9.82798166379715e-05, "loss": 0.009944040328264236, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00999, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.35668376088142395, "learning_rate": 9.819942181443002e-05, "loss": 0.004855391103774309, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00487, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.9442008137702942, "learning_rate": 9.811722928661392e-05, "loss": 0.015213963575661182, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.01533, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.12547080218791962, "learning_rate": 9.803324247488975e-05, "loss": 0.0013162264367565513, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00132, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 1.2091363668441772, "learning_rate": 9.794746487429161e-05, "loss": 0.013727689161896706, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01382, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.9757817387580872, "learning_rate": 9.785990005437554e-05, "loss": 0.01844210736453533, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01861, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 1.536257266998291, "learning_rate": 9.777055165907117e-05, "loss": 0.012463493272662163, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.01254, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 1.5637974739074707, "learning_rate": 9.767942340652993e-05, "loss": 0.031475603580474854, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.03198, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.5455516576766968, "learning_rate": 9.758651908897035e-05, "loss": 0.020683204755187035, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.0209, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.2210495471954346, "learning_rate": 9.749184257252033e-05, "loss": 0.030840927734971046, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.03132, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.5897301435470581, "learning_rate": 9.739539779705614e-05, "loss": 0.023160040378570557, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.02343, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.3296174108982086, "learning_rate": 9.729718877603861e-05, "loss": 0.008346345275640488, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00838, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 1.0745480060577393, "learning_rate": 9.719721959634592e-05, "loss": 0.031128475442528725, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03162, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.4855771064758301, "learning_rate": 9.709549441810375e-05, "loss": 0.016691725701093674, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01683, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.6925575733184814, "learning_rate": 9.699201747451195e-05, "loss": 0.0234430730342865, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02372, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.3683886229991913, "learning_rate": 9.688679307166854e-05, "loss": 0.013697060756385326, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01379, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 1.8865699768066406, "learning_rate": 9.677982558839042e-05, "loss": 0.022729959338903427, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02299, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.29908162355422974, "learning_rate": 9.66711194760312e-05, "loss": 0.010291436687111855, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01034, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 1.6500768661499023, "learning_rate": 9.656067925829593e-05, "loss": 0.0249391570687294, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.02525, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.41, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.677232563495636, "learning_rate": 9.644850953105288e-05, "loss": 0.025026176124811172, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.02534, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.440812349319458, "learning_rate": 9.633461496214225e-05, "loss": 0.014349604956805706, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01445, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.5943995714187622, "learning_rate": 9.621900029118195e-05, "loss": 0.026032745838165283, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.02637, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.38538894057273865, "learning_rate": 9.610167032937036e-05, "loss": 0.012495476752519608, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01257, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.2866664230823517, "learning_rate": 9.598262995928611e-05, "loss": 0.007073942106217146, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0071, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.38, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.15836192667484283, "learning_rate": 9.586188413468492e-05, "loss": 0.003585461527109146, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00359, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.41076672077178955, "learning_rate": 9.57394378802934e-05, "loss": 0.018675027415156364, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01885, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.4428853988647461, "learning_rate": 9.56152962916e-05, "loss": 0.016197098419070244, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.01633, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.3865810036659241, "learning_rate": 9.548946453464296e-05, "loss": 0.010973616503179073, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01103, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.7680912613868713, "learning_rate": 9.53619478457953e-05, "loss": 0.011722773313522339, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01179, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.7538973093032837, "learning_rate": 9.523275153154695e-05, "loss": 0.017707357183098793, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01787, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 1.6711620092391968, "learning_rate": 9.51018809682839e-05, "loss": 0.020698167383670807, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02091, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.8702117800712585, "learning_rate": 9.49693416020645e-05, "loss": 0.028852563351392746, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.02927, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.7649478912353516, "learning_rate": 9.483513894839276e-05, "loss": 0.006639844737946987, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00666, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.5647541880607605, "learning_rate": 9.469927859198888e-05, "loss": 0.012581500224769115, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01266, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.4410514533519745, "learning_rate": 9.456176618655689e-05, "loss": 0.018420841544866562, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.01859, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.5741907358169556, "learning_rate": 9.442260745454927e-05, "loss": 0.010290917940437794, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01034, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 2.4920763969421387, "learning_rate": 9.428180818692884e-05, "loss": 0.01673252135515213, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01687, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.46824702620506287, "learning_rate": 9.413937424292791e-05, "loss": 0.010418295860290527, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01047, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.67, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.49917715787887573, "learning_rate": 9.399531154980424e-05, "loss": 0.011245685629546642, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01131, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.5702803730964661, "learning_rate": 9.384962610259455e-05, "loss": 0.020356301218271255, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.02056, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.540846049785614, "learning_rate": 9.370232396386494e-05, "loss": 0.012266527861356735, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.01234, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.7793810963630676, "learning_rate": 9.355341126345868e-05, "loss": 0.020970115438103676, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02119, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.39841365814208984, "learning_rate": 9.340289419824107e-05, "loss": 0.007293989881873131, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00732, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.6273782849311829, "learning_rate": 9.325077903184159e-05, "loss": 0.0027987707871943712, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0028, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.09722045809030533, "learning_rate": 9.30970720943932e-05, "loss": 0.0012670346768572927, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00127, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.14121878147125244, "learning_rate": 9.2941779782269e-05, "loss": 0.002028010319918394, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00203, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.9108354449272156, "learning_rate": 9.278490855781596e-05, "loss": 0.023797553032636642, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02408, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 31.02, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.24121589958667755, "learning_rate": 9.262646494908604e-05, "loss": 0.0035530072636902332, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00356, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.8944887518882751, "learning_rate": 9.246645554956457e-05, "loss": 0.014964943751692772, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01508, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.8068950772285461, "learning_rate": 9.230488701789578e-05, "loss": 0.018670061603188515, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01885, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.5802752375602722, "learning_rate": 9.214176607760577e-05, "loss": 0.013859068043529987, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01396, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.8652616143226624, "learning_rate": 9.197709951682268e-05, "loss": 0.02518468163907528, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.0255, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 2.253959894180298, "learning_rate": 9.181089418799428e-05, "loss": 0.031588029116392136, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.03209, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.23236973583698273, "learning_rate": 9.164315700760271e-05, "loss": 0.006242041476070881, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00626, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 1.1648812294006348, "learning_rate": 9.147389495587671e-05, "loss": 0.01849639043211937, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01867, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 30.95, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.4124792814254761, "learning_rate": 9.130311507650116e-05, "loss": 0.014781644567847252, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01489, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 1.1504703760147095, "learning_rate": 9.113082447632394e-05, "loss": 0.02777012065052986, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.02816, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 32.94, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.571531355381012, "learning_rate": 9.09570303250602e-05, "loss": 0.024790536612272263, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.0251, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.19731058180332184, "learning_rate": 9.078173985499394e-05, "loss": 0.007044575177133083, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00707, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.5885587930679321, "learning_rate": 9.060496036067713e-05, "loss": 0.02042338065803051, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02063, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.6595909595489502, "learning_rate": 9.042669919862615e-05, "loss": 0.015881720930337906, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01601, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 1.077828288078308, "learning_rate": 9.024696378701557e-05, "loss": 0.031070906668901443, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.03156, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 2.388235092163086, "learning_rate": 9.006576160536948e-05, "loss": 0.016074448823928833, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.0162, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.4622959494590759, "learning_rate": 8.988310019425035e-05, "loss": 0.014777986332774162, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01489, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.14414255321025848, "learning_rate": 8.969898715494506e-05, "loss": 0.003839286044239998, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00385, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.5736926198005676, "learning_rate": 8.951343014914869e-05, "loss": 0.0190313421189785, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01921, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.33275526762008667, "learning_rate": 8.932643689864568e-05, "loss": 0.008519789204001427, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00856, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.5720345377922058, "learning_rate": 8.913801518498845e-05, "loss": 0.004408363252878189, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00442, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.17, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.4135957956314087, "learning_rate": 8.894817284917364e-05, "loss": 0.009598370641469955, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00964, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.2428351193666458, "learning_rate": 8.875691779131569e-05, "loss": 0.003992026671767235, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.004, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.61, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.3123396039009094, "learning_rate": 8.856425797031829e-05, "loss": 0.005327063612639904, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00534, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.4094838500022888, "learning_rate": 8.837020140354295e-05, "loss": 0.009061258286237717, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.0091, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.16227178275585175, "learning_rate": 8.817475616647554e-05, "loss": 0.002502303570508957, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00251, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 6.2613725662231445, "learning_rate": 8.797793039239017e-05, "loss": 0.00203700945712626, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00204, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.39469999074935913, "learning_rate": 8.777973227201069e-05, "loss": 0.007147412747144699, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00717, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 1.0135102272033691, "learning_rate": 8.758017005316988e-05, "loss": 0.008082596585154533, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00812, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.6063476204872131, "learning_rate": 8.737925204046629e-05, "loss": 0.009521201252937317, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00957, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 1.3042209148406982, "learning_rate": 8.717698659491851e-05, "loss": 0.03130755573511124, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0318, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.24, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 1.0105268955230713, "learning_rate": 8.697338213361735e-05, "loss": 0.02365228347480297, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02393, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.41111624240875244, "learning_rate": 8.676844712937552e-05, "loss": 0.006610711105167866, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00663, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.3596475422382355, "learning_rate": 8.656219011037509e-05, "loss": 0.00476874690502882, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00478, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 1.3601833581924438, "learning_rate": 8.63546196598125e-05, "loss": 0.00870994757860899, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00875, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.9145317077636719, "learning_rate": 8.614574441554145e-05, "loss": 0.03438536077737808, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.03498, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 1.1168012619018555, "learning_rate": 8.593557306971349e-05, "loss": 0.01852434128522873, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.0187, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.833202064037323, "learning_rate": 8.572411436841618e-05, "loss": 0.009514926001429558, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00956, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.21394768357276917, "learning_rate": 8.551137711130922e-05, "loss": 0.0066486382856965065, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00667, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.7765244245529175, "learning_rate": 8.529737015125824e-05, "loss": 0.013847503811120987, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01394, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.6788877844810486, "learning_rate": 8.508210239396639e-05, "loss": 0.017471041530370712, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01762, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.17106585204601288, "learning_rate": 8.486558279760375e-05, "loss": 0.004982849583029747, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.005, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.36792218685150146, "learning_rate": 8.464782037243449e-05, "loss": 0.010514025576412678, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01057, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.5413661003112793, "learning_rate": 8.442882418044202e-05, "loss": 0.013742422685027122, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01384, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.26433634757995605, "learning_rate": 8.420860333495179e-05, "loss": 0.009528635069727898, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00957, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 1.7420094013214111, "learning_rate": 8.398716700025208e-05, "loss": 0.005314864218235016, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00533, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.6060400009155273, "learning_rate": 8.376452439121266e-05, "loss": 0.020212136209011078, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.02042, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.63, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.3959150016307831, "learning_rate": 8.354068477290124e-05, "loss": 0.00989377312362194, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00994, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.2532685101032257, "learning_rate": 8.331565746019807e-05, "loss": 0.007109035272151232, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00713, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.41422078013420105, "learning_rate": 8.308945181740812e-05, "loss": 0.00821605697274208, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00825, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.42437371611595154, "learning_rate": 8.286207725787153e-05, "loss": 0.010428374633193016, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01048, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.6072356104850769, "learning_rate": 8.263354324357182e-05, "loss": 0.013942738994956017, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01404, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.2909337878227234, "learning_rate": 8.240385928474219e-05, "loss": 0.0071114227175712585, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00714, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.2351556420326233, "learning_rate": 8.217303493946967e-05, "loss": 0.0035386753734201193, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00354, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.5150942206382751, "learning_rate": 8.194107981329746e-05, "loss": 0.008464694023132324, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0085, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.21933956444263458, "learning_rate": 8.170800355882518e-05, "loss": 0.0016859541647136211, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00169, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.28, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.686940610408783, "learning_rate": 8.147381587530713e-05, "loss": 0.010961385443806648, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01102, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.8234176635742188, "learning_rate": 8.123852650824877e-05, "loss": 0.011313123628497124, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01138, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.45780348777770996, "learning_rate": 8.100214524900103e-05, "loss": 0.0035692225210368633, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00358, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.28213396668434143, "learning_rate": 8.076468193435301e-05, "loss": 0.0017856869380921125, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00179, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.38567906618118286, "learning_rate": 8.052614644612253e-05, "loss": 0.005117522552609444, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00513, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.20850537717342377, "learning_rate": 8.028654871074489e-05, "loss": 0.00270542548969388, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00271, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.1931680291891098, "learning_rate": 8.004589869885986e-05, "loss": 0.001737265381962061, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00174, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.9907969236373901, "learning_rate": 7.980420642489674e-05, "loss": 0.01618085987865925, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01631, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.10290444642305374, "learning_rate": 7.95614819466576e-05, "loss": 0.0006108212983235717, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00061, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.9275674819946289, "learning_rate": 7.931773536489872e-05, "loss": 0.014581063762307167, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01469, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.9798996448516846, "learning_rate": 7.907297682291035e-05, "loss": 0.006423726212233305, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00644, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 1.476671814918518, "learning_rate": 7.882721650609442e-05, "loss": 0.003532715607434511, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00354, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.7660159468650818, "learning_rate": 7.85804646415409e-05, "loss": 0.029062217101454735, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02949, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.09218309819698334, "learning_rate": 7.833273149760207e-05, "loss": 0.0009997149463742971, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.001, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 1.260366678237915, "learning_rate": 7.808402738346527e-05, "loss": 0.03378765657544136, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03436, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.39129698276519775, "learning_rate": 7.783436264872382e-05, "loss": 0.00922844372689724, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00927, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.2195906788110733, "learning_rate": 7.758374768294647e-05, "loss": 0.003479538019746542, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00349, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.4410147964954376, "learning_rate": 7.733219291524489e-05, "loss": 0.009434984996914864, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00948, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.22936853766441345, "learning_rate": 7.707970881383977e-05, "loss": 0.0036677452735602856, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00367, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.4793826639652252, "learning_rate": 7.682630588562518e-05, "loss": 0.014816081151366234, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01493, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.518409013748169, "learning_rate": 7.657199467573129e-05, "loss": 0.011389249935746193, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01145, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.7315135598182678, "learning_rate": 7.631678576708561e-05, "loss": 0.008425935171544552, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00846, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.2665201425552368, "learning_rate": 7.606068977997255e-05, "loss": 0.004567623138427734, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00458, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 2.1971797943115234, "learning_rate": 7.580371737159148e-05, "loss": 0.017572522163391113, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01773, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.7911526560783386, "learning_rate": 7.554587923561324e-05, "loss": 0.006907682865858078, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00693, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.6504429578781128, "learning_rate": 7.528718610173511e-05, "loss": 0.012268126010894775, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01234, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.74, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.32485631108283997, "learning_rate": 7.502764873523431e-05, "loss": 0.010928580537438393, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01099, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 1.451930046081543, "learning_rate": 7.476727793652011e-05, "loss": 0.006648082751780748, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00667, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 1.151189923286438, "learning_rate": 7.450608454068415e-05, "loss": 0.013467690907418728, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01356, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.374714732170105, "learning_rate": 7.424407941704987e-05, "loss": 0.010055478662252426, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01011, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 2.5658504962921143, "learning_rate": 7.398127346871986e-05, "loss": 0.03230627626180649, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.03283, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.3449725806713104, "learning_rate": 7.371767763212238e-05, "loss": 0.008773802779614925, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00881, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 1.906986117362976, "learning_rate": 7.345330287655617e-05, "loss": 0.014551490545272827, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01466, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.6069923043251038, "learning_rate": 7.31881602037339e-05, "loss": 0.014878768473863602, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01499, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.2538802921772003, "learning_rate": 7.29222606473245e-05, "loss": 0.004948832094669342, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00496, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.303553968667984, "learning_rate": 7.265561527249383e-05, "loss": 0.004754569847136736, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00477, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.4976601302623749, "learning_rate": 7.238823517544436e-05, "loss": 0.010160710662603378, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01021, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.277299165725708, "learning_rate": 7.212013148295333e-05, "loss": 0.004368964582681656, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00438, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.2698659896850586, "learning_rate": 7.185131535190975e-05, "loss": 0.0062669795006513596, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00629, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.7640749216079712, "learning_rate": 7.158179796885005e-05, "loss": 0.00835468340665102, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00839, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.20470944046974182, "learning_rate": 7.131159054949273e-05, "loss": 0.005909225437790155, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00593, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 9.618382453918457, "learning_rate": 7.104070433827139e-05, "loss": 0.0024531008675694466, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00246, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.8679444789886475, "learning_rate": 7.076915060786705e-05, "loss": 0.013406777754426003, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0135, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.7233160734176636, "learning_rate": 7.049694065873882e-05, "loss": 0.011322369799017906, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01139, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.33370518684387207, "learning_rate": 7.022408581865382e-05, "loss": 0.005950187798589468, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00597, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.09047522395849228, "learning_rate": 6.99505974422157e-05, "loss": 0.0007227606838569045, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00072, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.18869127333164215, "learning_rate": 6.967648691039213e-05, "loss": 0.002227586228400469, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00223, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.13700009882450104, "learning_rate": 6.940176563004123e-05, "loss": 0.002167265396565199, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00217, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.7476372718811035, "learning_rate": 6.912644503343682e-05, "loss": 0.012073586694896221, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01215, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.6000645756721497, "learning_rate": 6.885053657779273e-05, "loss": 0.02252592146396637, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02278, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.38331103324890137, "learning_rate": 6.857405174478604e-05, "loss": 0.0036613992415368557, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00367, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.7047562599182129, "learning_rate": 6.82970020400792e-05, "loss": 0.017694411799311638, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01785, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.11769180744886398, "learning_rate": 6.801939899284132e-05, "loss": 0.0021647342946380377, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00217, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.3932294547557831, "learning_rate": 6.774125415526827e-05, "loss": 0.005032756831496954, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00505, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 1.6520779132843018, "learning_rate": 6.746257910210214e-05, "loss": 0.005828537046909332, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00585, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.6707406044006348, "learning_rate": 6.718338543014937e-05, "loss": 0.016984354704618454, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.01713, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.78, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.3941832482814789, "learning_rate": 6.69036847577983e-05, "loss": 0.0057555451057851315, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00577, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 1.6337946653366089, "learning_rate": 6.662348872453553e-05, "loss": 0.06337247788906097, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.06542, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.13002485036849976, "learning_rate": 6.63428089904618e-05, "loss": 0.0019444981589913368, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00195, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.23482702672481537, "learning_rate": 6.60616572358065e-05, "loss": 0.005110039841383696, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00512, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.20000001788139343, "learning_rate": 6.578004516044172e-05, "loss": 0.00558443833142519, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0056, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.33824580907821655, "learning_rate": 6.549798448339548e-05, "loss": 0.010158074088394642, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.01021, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.1845032274723053, "learning_rate": 6.521548694236384e-05, "loss": 0.0037364959716796875, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00374, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.3662629723548889, "learning_rate": 6.493256429322259e-05, "loss": 0.012112480588257313, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01219, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.6794489622116089, "learning_rate": 6.464922830953799e-05, "loss": 0.029255975037813187, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.02969, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 2.283385753631592, "learning_rate": 6.436549078207688e-05, "loss": 0.02822192944586277, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02862, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.5736503601074219, "learning_rate": 6.408136351831592e-05, "loss": 0.017562061548233032, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01772, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.07728739827871323, "learning_rate": 6.379685834195036e-05, "loss": 0.0022523219231516123, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00225, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.2413042038679123, "learning_rate": 6.351198709240186e-05, "loss": 0.009340126067399979, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00938, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.9, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.17655512690544128, "learning_rate": 6.32267616243259e-05, "loss": 0.009295165538787842, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00934, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.54, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.1462055891752243, "learning_rate": 6.294119380711849e-05, "loss": 0.00600145710632205, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00602, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.5610263347625732, "learning_rate": 6.265529552442209e-05, "loss": 0.018549544736742973, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01872, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.1185809001326561, "learning_rate": 6.236907867363127e-05, "loss": 0.0033995567355304956, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00341, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.13333943486213684, "learning_rate": 6.208255516539749e-05, "loss": 0.004120497032999992, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00413, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.12274736911058426, "learning_rate": 6.179573692313344e-05, "loss": 0.0034421663731336594, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00345, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.1564289629459381, "learning_rate": 6.150863588251694e-05, "loss": 0.004175345413386822, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00418, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.5057165622711182, "learning_rate": 6.122126399099419e-05, "loss": 0.01389961875975132, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.014, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 1.4075968265533447, "learning_rate": 6.0933633207282615e-05, "loss": 0.034657664597034454, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.03527, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.20491240918636322, "learning_rate": 6.064575550087316e-05, "loss": 0.003817569464445114, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00382, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.5278146266937256, "learning_rate": 6.0357642851532245e-05, "loss": 0.004187127109616995, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0042, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.31802991032600403, "learning_rate": 6.0069307248803294e-05, "loss": 0.003848882159218192, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00386, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.2332313507795334, "learning_rate": 5.9780760691507635e-05, "loss": 0.00399994058534503, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00401, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.8588718175888062, "learning_rate": 5.9492015187245334e-05, "loss": 0.0015302450628951192, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00153, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.05017514154314995, "learning_rate": 5.920308275189541e-05, "loss": 0.00032446341356262565, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00032, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.4395122230052948, "learning_rate": 5.8913975409115874e-05, "loss": 0.017047671601176262, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01719, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.049575477838516235, "learning_rate": 5.8624705189843395e-05, "loss": 0.0004300390719436109, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00043, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.1624041050672531, "learning_rate": 5.833528413179249e-05, "loss": 0.0005117984837852418, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00051, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.34115126729011536, "learning_rate": 5.80457242789548e-05, "loss": 0.009314409457147121, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00936, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.25679922103881836, "learning_rate": 5.77560376810977e-05, "loss": 0.0028868727385997772, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00289, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.1863754540681839, "learning_rate": 5.7466236393263005e-05, "loss": 0.0009845286840572953, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00099, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.6423511505126953, "learning_rate": 5.717633247526522e-05, "loss": 0.011791419237852097, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01186, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.2335442304611206, "learning_rate": 5.688633799118971e-05, "loss": 0.0028811718802899122, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00289, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.29636526107788086, "learning_rate": 5.659626500889066e-05, "loss": 0.006674672476947308, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0067, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.22831514477729797, "learning_rate": 5.6306125599488905e-05, "loss": 0.0023422972299158573, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00235, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.3985448181629181, "learning_rate": 5.601593183686955e-05, "loss": 0.005415226332843304, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00543, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.31169193983078003, "learning_rate": 5.572569579717961e-05, "loss": 0.006436801049858332, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00646, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.03327668085694313, "learning_rate": 5.543542955832538e-05, "loss": 0.0005451483302749693, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00055, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.52, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.42802318930625916, "learning_rate": 5.514514519946986e-05, "loss": 0.004677595570683479, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00469, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.36615973711013794, "learning_rate": 5.485485480053015e-05, "loss": 0.012691331095993519, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.01277, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.83, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.2485608607530594, "learning_rate": 5.4564570441674645e-05, "loss": 0.002122094389051199, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00212, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.14074647426605225, "learning_rate": 5.42743042028204e-05, "loss": 0.0009143606293946505, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00091, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.79, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.20445789396762848, "learning_rate": 5.3984068163130464e-05, "loss": 0.002428838051855564, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00243, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.5443958044052124, "learning_rate": 5.369387440051111e-05, "loss": 0.01616598851978779, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0163, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.04622013494372368, "learning_rate": 5.340373499110935e-05, "loss": 0.0007802956970408559, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00078, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.05457351729273796, "learning_rate": 5.3113662008810304e-05, "loss": 0.0007781478925608099, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00078, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.06809470057487488, "learning_rate": 5.282366752473479e-05, "loss": 0.0008410143782384694, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00084, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.045030321925878525, "learning_rate": 5.2533763606737005e-05, "loss": 0.0009256709599867463, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00093, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.22, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.3253517746925354, "learning_rate": 5.224396231890232e-05, "loss": 0.006005933973938227, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00602, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.32163333892822266, "learning_rate": 5.195427572104522e-05, "loss": 0.005790356080979109, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00581, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.3931090831756592, "learning_rate": 5.166471586820751e-05, "loss": 0.007895519025623798, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00793, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.3888826370239258, "learning_rate": 5.1375294810156615e-05, "loss": 0.013600911945104599, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01369, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.1380021572113037, "learning_rate": 5.1086024590884144e-05, "loss": 0.0015287620481103659, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00153, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.198189377784729, "learning_rate": 5.079691724810461e-05, "loss": 0.003010803833603859, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00302, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.040619101375341415, "learning_rate": 5.0507984812754684e-05, "loss": 0.000593566510360688, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00059, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.6, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.12623795866966248, "learning_rate": 5.021923930849237e-05, "loss": 0.0012465681647881866, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00125, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.06826923042535782, "learning_rate": 4.99306927511967e-05, "loss": 0.0006613734294660389, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00066, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.2856789231300354, "learning_rate": 4.964235714846775e-05, "loss": 0.004899258725345135, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00491, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.09510622918605804, "learning_rate": 4.9354244499126866e-05, "loss": 0.0007005144143477082, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.0007, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.08008120208978653, "learning_rate": 4.90663667927174e-05, "loss": 0.00029125864966772497, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00029, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.04350510612130165, "learning_rate": 4.877873600900581e-05, "loss": 0.00044145819265395403, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00044, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.62, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.6371538639068604, "learning_rate": 4.849136411748306e-05, "loss": 0.0012232076842337847, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00122, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.24129775166511536, "learning_rate": 4.8204263076866574e-05, "loss": 0.011151394806802273, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01121, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.52, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.04135918244719505, "learning_rate": 4.791744483460251e-05, "loss": 0.000246158946538344, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00025, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.04536788910627365, "learning_rate": 4.7630921326368736e-05, "loss": 0.0003238982753828168, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00032, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 19.94, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 1.3142327070236206, "learning_rate": 4.7344704475577916e-05, "loss": 0.01239306852221489, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.01247, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 1.9168639183044434, "learning_rate": 4.705880619288153e-05, "loss": 0.01447539683431387, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.01458, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 21.94, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.1793145090341568, "learning_rate": 4.677323837567412e-05, "loss": 0.001607376616448164, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00161, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.21612292528152466, "learning_rate": 4.6488012907598146e-05, "loss": 0.0003686543495859951, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00037, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.15809769928455353, "learning_rate": 4.620314165804964e-05, "loss": 0.002215825952589512, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00222, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.4193531274795532, "learning_rate": 4.591863648168407e-05, "loss": 0.008472323417663574, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00851, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.08075176924467087, "learning_rate": 4.5634509217923135e-05, "loss": 0.0005553544033318758, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00056, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.11020452529191971, "learning_rate": 4.535077169046201e-05, "loss": 0.0010436817537993193, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00104, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.6850148439407349, "learning_rate": 4.506743570677743e-05, "loss": 0.017507392913103104, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01766, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.22, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.1557648628950119, "learning_rate": 4.478451305763618e-05, "loss": 0.0015634099254384637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00156, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.1412716507911682, "learning_rate": 4.450201551660454e-05, "loss": 0.001437606057152152, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00144, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.49981996417045593, "learning_rate": 4.4219954839558276e-05, "loss": 0.005886933766305447, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0059, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.19367864727973938, "learning_rate": 4.393834276419352e-05, "loss": 0.0047086565755307674, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00472, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.46, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.16504675149917603, "learning_rate": 4.36571910095382e-05, "loss": 0.0007262254366651177, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00073, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.1498098373413086, "learning_rate": 4.337651127546448e-05, "loss": 0.0006320225656963885, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00063, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.1048150435090065, "learning_rate": 4.3096315242201736e-05, "loss": 0.0012588162207975984, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00126, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.038675930351018906, "learning_rate": 4.2816614569850635e-05, "loss": 0.00016195193165913224, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00016, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.6870748996734619, "learning_rate": 4.2537420897897864e-05, "loss": 0.007371625863015652, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0074, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.03561795875430107, "learning_rate": 4.225874584473174e-05, "loss": 0.0004196897498331964, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00042, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.055394649505615234, "learning_rate": 4.19806010071587e-05, "loss": 0.0005032906192354858, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0005, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.4164913296699524, "learning_rate": 4.170299795992081e-05, "loss": 0.01414618268609047, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.01425, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.7483956217765808, "learning_rate": 4.142594825521398e-05, "loss": 0.0016437128651887178, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00165, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.13886658847332, "learning_rate": 4.114946342220728e-05, "loss": 0.0004101275699213147, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00041, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.054154932498931885, "learning_rate": 4.087355496656321e-05, "loss": 0.0006212553125806153, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00062, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.2857692241668701, "learning_rate": 4.05982343699588e-05, "loss": 0.0016585109988227487, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00166, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.2486022412776947, "learning_rate": 4.0323513089607876e-05, "loss": 0.0035688348580151796, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00358, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.17051135003566742, "learning_rate": 4.004940255778431e-05, "loss": 0.002337381709367037, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00234, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.3675305247306824, "learning_rate": 3.977591418134619e-05, "loss": 0.005339859984815121, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00535, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.2916986644268036, "learning_rate": 3.95030593412612e-05, "loss": 0.004081412684172392, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00409, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.4349796175956726, "learning_rate": 3.923084939213296e-05, "loss": 0.0045698839239776134, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00458, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.1905355453491211, "learning_rate": 3.895929566172861e-05, "loss": 0.0029714226257056, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00298, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.5870007276535034, "learning_rate": 3.868840945050728e-05, "loss": 0.006435167510062456, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00646, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.46, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.031167343258857727, "learning_rate": 3.841820203114995e-05, "loss": 0.00019888720999006182, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0002, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.15903304517269135, "learning_rate": 3.814868464809027e-05, "loss": 0.0020774153526872396, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00208, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.023943398147821426, "learning_rate": 3.787986851704667e-05, "loss": 0.00012388546019792557, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00012, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.010502259247004986, "learning_rate": 3.7611764824555654e-05, "loss": 0.00017548314644955099, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00018, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 30.85, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.03495743125677109, "learning_rate": 3.734438472750619e-05, "loss": 0.00034388076164759696, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00034, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.013371769338846207, "learning_rate": 3.707773935267552e-05, "loss": 0.00018656470638234168, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00019, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.022662173956632614, "learning_rate": 3.68118397962661e-05, "loss": 0.0002498112735338509, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00025, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.038852933794260025, "learning_rate": 3.654669712344384e-05, "loss": 0.00039461010601371527, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00039, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.57, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.04052181914448738, "learning_rate": 3.628232236787763e-05, "loss": 0.0005348794511519372, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00054, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.5525831580162048, "learning_rate": 3.6018726531280144e-05, "loss": 0.005064015742391348, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00508, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.13009344041347504, "learning_rate": 3.575592058295017e-05, "loss": 0.0014344020746648312, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00144, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.048520784825086594, "learning_rate": 3.549391545931585e-05, "loss": 0.0001677365362411365, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00017, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.10236997902393341, "learning_rate": 3.5232722063479914e-05, "loss": 0.0010845332872122526, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00109, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.1809798628091812, "learning_rate": 3.49723512647657e-05, "loss": 0.002675148192793131, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00268, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.14033831655979156, "learning_rate": 3.471281389826491e-05, "loss": 0.0007573956972919405, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00076, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.09931791573762894, "learning_rate": 3.4454120764386764e-05, "loss": 0.0007360494928434491, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00074, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.052966371178627014, "learning_rate": 3.4196282628408526e-05, "loss": 0.0005538966506719589, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00055, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.08541939407587051, "learning_rate": 3.3939310220027456e-05, "loss": 0.0009438276174478233, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00094, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.2350178211927414, "learning_rate": 3.3683214232914404e-05, "loss": 0.0015610259724780917, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00156, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.008908914402127266, "learning_rate": 3.342800532426873e-05, "loss": 7.28988234186545e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00007, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.10018748790025711, "learning_rate": 3.317369411437484e-05, "loss": 0.0009240458603017032, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00092, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.10880228877067566, "learning_rate": 3.292029118616024e-05, "loss": 0.0009955165442079306, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.001, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.16797584295272827, "learning_rate": 3.266780708475511e-05, "loss": 0.00212887697853148, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00213, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.3653261065483093, "learning_rate": 3.241625231705354e-05, "loss": 0.003090285463258624, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0031, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.09819911420345306, "learning_rate": 3.216563735127618e-05, "loss": 0.0006902081076987088, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00069, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.34457531571388245, "learning_rate": 3.191597261653475e-05, "loss": 0.0008856031927280128, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00089, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 1.2643779516220093, "learning_rate": 3.166726850239794e-05, "loss": 0.01733694225549698, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01749, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.5604080557823181, "learning_rate": 3.141953535845912e-05, "loss": 0.002659569261595607, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00266, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.04489889368414879, "learning_rate": 3.11727834939056e-05, "loss": 0.0002243104245280847, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00022, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.2954617738723755, "learning_rate": 3.092702317708967e-05, "loss": 0.006130777765065432, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00615, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.0030694541055709124, "learning_rate": 3.0682264635101276e-05, "loss": 2.4017694158828817e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00002, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.037856340408325195, "learning_rate": 3.0438518053342407e-05, "loss": 0.00023381944629363716, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00023, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.17, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.1950075626373291, "learning_rate": 3.0195793575103266e-05, "loss": 0.0012865000171586871, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00129, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.2, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.23903191089630127, "learning_rate": 2.9954101301140146e-05, "loss": 0.0017133182846009731, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00171, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.1501404196023941, "learning_rate": 2.9713451289255123e-05, "loss": 0.0013980502262711525, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.0014, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.005602911114692688, "learning_rate": 2.9473853553877484e-05, "loss": 3.3548218198120594e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00003, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.03793817758560181, "learning_rate": 2.9235318065647e-05, "loss": 0.0003298196825198829, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00033, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.0022431630641222, "learning_rate": 2.8997854750998964e-05, "loss": 2.8589034627657384e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00003, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.17198579013347626, "learning_rate": 2.8761473491751258e-05, "loss": 0.000326915382174775, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00033, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.31277862191200256, "learning_rate": 2.8526184124692883e-05, "loss": 0.0025436156429350376, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00255, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.34, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.154794842004776, "learning_rate": 2.829199644117484e-05, "loss": 0.0014764989027753472, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00148, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.024617183953523636, "learning_rate": 2.8058920186702553e-05, "loss": 0.00023092771880328655, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00023, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.0025126859545707703, "learning_rate": 2.782696506053033e-05, "loss": 3.5130899050273e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00004, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.4731985330581665, "learning_rate": 2.7596140715257824e-05, "loss": 0.005484737455844879, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.0055, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0559220165014267, "learning_rate": 2.7366456756428184e-05, "loss": 0.0003944181080441922, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00039, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.27162396907806396, "learning_rate": 2.7137922742128486e-05, "loss": 0.005523894913494587, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00554, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.008168015629053116, "learning_rate": 2.691054818259188e-05, "loss": 7.397416629828513e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00007, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.03337949141860008, "learning_rate": 2.6684342539801933e-05, "loss": 9.702846728032455e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.0001, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.73, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.15445096790790558, "learning_rate": 2.645931522709877e-05, "loss": 0.0009240603540092707, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00092, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.5359209179878235, "learning_rate": 2.6235475608787365e-05, "loss": 0.014039446599781513, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.01414, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.11708246171474457, "learning_rate": 2.6012832999747916e-05, "loss": 0.0010296297259628773, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00103, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.22, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.6646474599838257, "learning_rate": 2.579139666504821e-05, "loss": 0.01486277673393488, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01497, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.09990301728248596, "learning_rate": 2.557117581955798e-05, "loss": 0.00031419514562003314, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00031, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.003383894916623831, "learning_rate": 2.5352179627565532e-05, "loss": 6.680995284114033e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00007, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.25627484917640686, "learning_rate": 2.5134417202396277e-05, "loss": 0.001967544900253415, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00197, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.08340954035520554, "learning_rate": 2.491789760603361e-05, "loss": 0.000622199266217649, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00062, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.02069864235818386, "learning_rate": 2.4702629848741764e-05, "loss": 0.0003160611377097666, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00032, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.3115765452384949, "learning_rate": 2.4488622888690785e-05, "loss": 0.005425162613391876, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00544, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.03467710316181183, "learning_rate": 2.427588563158384e-05, "loss": 0.0003728372685145587, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00037, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.1006288006901741, "learning_rate": 2.406442693028651e-05, "loss": 0.0015020224964246154, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.0015, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.014766009524464607, "learning_rate": 2.3854255584458547e-05, "loss": 0.0002054517244687304, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00021, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.57, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.0996914803981781, "learning_rate": 2.3645380340187508e-05, "loss": 0.0010447031818330288, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00105, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.0755075141787529, "learning_rate": 2.3437809889624914e-05, "loss": 0.001047295518219471, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00105, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.14409306645393372, "learning_rate": 2.3231552870624487e-05, "loss": 0.0008429547888226807, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00084, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.2477526068687439, "learning_rate": 2.3026617866382657e-05, "loss": 0.0012035734253004193, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.0012, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.026689043268561363, "learning_rate": 2.2823013405081507e-05, "loss": 0.00027347676223143935, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00027, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.01691209338605404, "learning_rate": 2.2620747959533722e-05, "loss": 8.836232882458717e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00009, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.64, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.07997870445251465, "learning_rate": 2.2419829946830123e-05, "loss": 0.0008721392368897796, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00087, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.2840462625026703, "learning_rate": 2.2220267727989325e-05, "loss": 0.00047142597031779587, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00047, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.7352865934371948, "learning_rate": 2.202206960760984e-05, "loss": 0.012425636872649193, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0125, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.04582185670733452, "learning_rate": 2.182524383352446e-05, "loss": 0.0005812362069264054, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00058, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.4995034337043762, "learning_rate": 2.1629798596457056e-05, "loss": 0.007702314294874668, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00773, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.020584356039762497, "learning_rate": 2.1435742029681725e-05, "loss": 0.0001667431934038177, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00017, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.00823939684778452, "learning_rate": 2.124308220868431e-05, "loss": 6.475761620095e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00006, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.061513502150774, "learning_rate": 2.105182715082638e-05, "loss": 0.0012119438033550978, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00121, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.032238274812698364, "learning_rate": 2.0861984815011552e-05, "loss": 0.0002466488513164222, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00025, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.03830055892467499, "learning_rate": 2.0673563101354323e-05, "loss": 0.00064308219589293, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00064, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.27470192313194275, "learning_rate": 2.0486569850851317e-05, "loss": 0.0037543869111686945, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00376, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.20996904373168945, "learning_rate": 2.0301012845054956e-05, "loss": 0.002677955897524953, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00268, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.38, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.015847615897655487, "learning_rate": 2.011689980574966e-05, "loss": 0.0002680838806554675, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00027, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.13935989141464233, "learning_rate": 1.993423839463052e-05, "loss": 0.0020974043291062117, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0021, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.013939899392426014, "learning_rate": 1.975303621298445e-05, "loss": 0.00018884852761402726, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00019, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.010106701403856277, "learning_rate": 1.957330080137385e-05, "loss": 0.00014173910312820226, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00014, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.03762160614132881, "learning_rate": 1.9395039639322864e-05, "loss": 0.000409349100664258, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00041, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.016227195039391518, "learning_rate": 1.9218260145006073e-05, "loss": 0.00019703531870618463, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.0002, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.05364619940519333, "learning_rate": 1.904296967493982e-05, "loss": 0.0008058860548771918, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00081, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.6282008290290833, "learning_rate": 1.8869175523676064e-05, "loss": 0.005147240124642849, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00516, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.0591219998896122, "learning_rate": 1.869688492349885e-05, "loss": 0.0007890574051998556, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00079, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.061940424144268036, "learning_rate": 1.85261050441233e-05, "loss": 0.0005965966265648603, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.0006, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.3280227482318878, "learning_rate": 1.8356842992397304e-05, "loss": 0.0059966156259179115, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00601, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.030033394694328308, "learning_rate": 1.8189105812005714e-05, "loss": 0.00026904212427325547, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00027, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.04036545008420944, "learning_rate": 1.802290048317732e-05, "loss": 0.0004486394173000008, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00045, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.011995392851531506, "learning_rate": 1.785823392239424e-05, "loss": 0.00016477004101034254, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00016, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.015974095091223717, "learning_rate": 1.7695112982104225e-05, "loss": 0.0002275579608976841, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00023, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.042813740670681, "learning_rate": 1.7533544450435433e-05, "loss": 0.0002771879662759602, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00028, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.024679042398929596, "learning_rate": 1.7373535050913946e-05, "loss": 0.0001260364952031523, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00013, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.5439828038215637, "learning_rate": 1.721509144218405e-05, "loss": 0.008165246807038784, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.0082, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.8, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.0370185449719429, "learning_rate": 1.705822021773101e-05, "loss": 0.0003393165534362197, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00034, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.4611383378505707, "learning_rate": 1.69029279056068e-05, "loss": 0.017980240285396576, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01814, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.014943188056349754, "learning_rate": 1.6749220968158415e-05, "loss": 6.63218816043809e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00007, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.012720294296741486, "learning_rate": 1.659710580175893e-05, "loss": 0.00015205545059870929, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00015, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 32.07, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.13785763084888458, "learning_rate": 1.644658873654133e-05, "loss": 0.00144623639062047, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00145, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.39, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.006861523725092411, "learning_rate": 1.629767603613508e-05, "loss": 0.00011123051081085578, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00011, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.2759925127029419, "learning_rate": 1.615037389740547e-05, "loss": 0.0038564258720725775, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00386, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.22087515890598297, "learning_rate": 1.600468845019576e-05, "loss": 0.0027728041168302298, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00278, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.004629611037671566, "learning_rate": 1.5860625757072092e-05, "loss": 6.992170528974384e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00007, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.05647851526737213, "learning_rate": 1.571819181307116e-05, "loss": 0.0003965786308981478, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.0004, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.027850763872265816, "learning_rate": 1.557739254545075e-05, "loss": 0.0003817995311692357, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00038, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.68, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.07775162905454636, "learning_rate": 1.543823381344311e-05, "loss": 0.0007786316564306617, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00078, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 31.13, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.24506999552249908, "learning_rate": 1.5300721408011114e-05, "loss": 0.002766917459666729, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00277, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.11139097064733505, "learning_rate": 1.5164861051607254e-05, "loss": 0.0007836521253921092, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.00078, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.019544478505849838, "learning_rate": 1.5030658397935521e-05, "loss": 0.00010838037269422784, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00011, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.02996714785695076, "learning_rate": 1.4898119031716104e-05, "loss": 0.0003544097999110818, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00035, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.007755239959806204, "learning_rate": 1.476724846845306e-05, "loss": 0.00011848683061543852, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00012, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.01187409833073616, "learning_rate": 1.463805215420471e-05, "loss": 0.00015462914598174393, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00015, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.02145065739750862, "learning_rate": 1.451053546535705e-05, "loss": 0.00018121419998351485, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00018, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.015912197530269623, "learning_rate": 1.438470370840001e-05, "loss": 0.00020458988728933036, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0002, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.0968913808465004, "learning_rate": 1.4260562119706606e-05, "loss": 0.0012479525757953525, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00125, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.02829655259847641, "learning_rate": 1.413811586531508e-05, "loss": 0.0003127091040369123, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00031, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.04508420452475548, "learning_rate": 1.4017370040713884e-05, "loss": 0.00045831367606297135, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00046, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.04927334934473038, "learning_rate": 1.3898329670629645e-05, "loss": 0.0006846758769825101, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00068, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 20.98, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.003517783246934414, "learning_rate": 1.3780999708818058e-05, "loss": 4.540931695373729e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00005, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.008153259754180908, "learning_rate": 1.3665385037857758e-05, "loss": 9.992026025429368e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.0001, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.03480210527777672, "learning_rate": 1.3551490468947126e-05, "loss": 0.00048806017730385065, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00049, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.3702182173728943, "learning_rate": 1.3439320741704075e-05, "loss": 0.003967836033552885, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00398, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.035434260964393616, "learning_rate": 1.3328880523968808e-05, "loss": 0.00045934616355225444, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00046, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.10519666224718094, "learning_rate": 1.3220174411609587e-05, "loss": 0.00038070077425800264, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00038, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.12639182806015015, "learning_rate": 1.3113206928331471e-05, "loss": 0.0007420229958370328, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00074, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.002904025139287114, "learning_rate": 1.300798252548806e-05, "loss": 4.279932181816548e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00004, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.004548432771116495, "learning_rate": 1.2904505581896265e-05, "loss": 6.924309127498418e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00007, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.0815773606300354, "learning_rate": 1.2802780403654082e-05, "loss": 0.0009746674913913012, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00098, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.014013510197401047, "learning_rate": 1.2702811223961408e-05, "loss": 0.00012820123811252415, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00013, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.012004408985376358, "learning_rate": 1.2604602202943861e-05, "loss": 0.00013364071492105722, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00013, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.02049950510263443, "learning_rate": 1.2508157427479686e-05, "loss": 0.00019200546375941485, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00019, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.07431647926568985, "learning_rate": 1.2413480911029655e-05, "loss": 0.0010132731404155493, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00101, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.008185718208551407, "learning_rate": 1.2320576593470082e-05, "loss": 0.0001087641721824184, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00011, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.07110211253166199, "learning_rate": 1.2229448340928828e-05, "loss": 0.0006274295737966895, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00063, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.0328889898955822, "learning_rate": 1.2140099945624458e-05, "loss": 0.0003421592991799116, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00034, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.2913395166397095, "learning_rate": 1.205253512570841e-05, "loss": 0.0022475982550531626, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00225, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 1.1127585172653198, "learning_rate": 1.1966757525110255e-05, "loss": 0.004208527505397797, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00422, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 30.8, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.3262764513492584, "learning_rate": 1.1882770713386095e-05, "loss": 0.005743140354752541, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00576, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.03095809370279312, "learning_rate": 1.180057818556998e-05, "loss": 0.0002009888703469187, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.0002, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.03716304153203964, "learning_rate": 1.1720183362028494e-05, "loss": 0.00013607167056761682, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00014, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.15490928292274475, "learning_rate": 1.1641589588318387e-05, "loss": 0.0008924140129238367, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00089, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.0017065296415239573, "learning_rate": 1.1564800135047418e-05, "loss": 1.8099377484759316e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00002, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.25381261110305786, "learning_rate": 1.148981819773816e-05, "loss": 0.0020984376315027475, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.0021, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.004274156875908375, "learning_rate": 1.1416646896695086e-05, "loss": 4.343297041486949e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00004, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.22789233922958374, "learning_rate": 1.1345289276874717e-05, "loss": 0.002067035064101219, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00207, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.009155255742371082, "learning_rate": 1.1275748307758873e-05, "loss": 7.678172551095486e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00008, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.012658460065722466, "learning_rate": 1.1208026883231147e-05, "loss": 9.109734673984349e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00009, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.004959877580404282, "learning_rate": 1.1142127821456433e-05, "loss": 8.5059815319255e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00009, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.01542067714035511, "learning_rate": 1.1078053864763674e-05, "loss": 0.00015253681340254843, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00015, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.3618672490119934, "learning_rate": 1.1015807679531756e-05, "loss": 0.002783063566312194, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00279, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.8618771433830261, "learning_rate": 1.0955391856078528e-05, "loss": 0.007333735469728708, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00736, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.09305798262357712, "learning_rate": 1.0896808908553007e-05, "loss": 0.0008098330581560731, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00081, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.26874008774757385, "learning_rate": 1.0840061274830763e-05, "loss": 0.0027115067932754755, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00272, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.01213931292295456, "learning_rate": 1.0785151316412473e-05, "loss": 0.00013484009832609445, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00013, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.10388921201229095, "learning_rate": 1.0732081318325639e-05, "loss": 0.0006787723978050053, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00068, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.01165830995887518, "learning_rate": 1.0680853489029501e-05, "loss": 8.822458039503545e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00009, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.7645137906074524, "learning_rate": 1.0631469960323152e-05, "loss": 0.007718742825090885, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00775, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.59, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.00700168963521719, "learning_rate": 1.0583932787256783e-05, "loss": 7.104189717210829e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00007, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.010428660549223423, "learning_rate": 1.0538243948046206e-05, "loss": 0.00012555076682474464, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00013, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.018040992319583893, "learning_rate": 1.0494405343990523e-05, "loss": 0.00017645448679104447, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00018, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.0795431137084961, "learning_rate": 1.0452418799392985e-05, "loss": 0.0004504769167397171, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00045, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.16452012956142426, "learning_rate": 1.0412286061485102e-05, "loss": 0.004217743873596191, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00423, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.1048925444483757, "learning_rate": 1.03740088003539e-05, "loss": 0.0005231528193689883, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00052, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.54, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.4650193452835083, "learning_rate": 1.0337588608872463e-05, "loss": 0.005999073851853609, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00602, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 31.07, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.06905589252710342, "learning_rate": 1.0303027002633622e-05, "loss": 0.00044624562724493444, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00045, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.002172560663893819, "learning_rate": 1.0270325419886884e-05, "loss": 2.402822792646475e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00002, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.84, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.16266341507434845, "learning_rate": 1.0239485221478599e-05, "loss": 0.0010158825898543, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00102, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.001998042920604348, "learning_rate": 1.0210507690795292e-05, "loss": 4.1451421566307545e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00004, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.254403293132782, "learning_rate": 1.0183394033710305e-05, "loss": 0.003670802805572748, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00368, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.09734537452459335, "learning_rate": 1.0158145378533583e-05, "loss": 0.0008671102696098387, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00087, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.07713494449853897, "learning_rate": 1.0134762775964726e-05, "loss": 0.0006400190177373588, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00064, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.09956923872232437, "learning_rate": 1.0113247199049278e-05, "loss": 0.0007513290620408952, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00075, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.04724310711026192, "learning_rate": 1.0093599543138205e-05, "loss": 0.0008900416432879865, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00089, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.17036564648151398, "learning_rate": 1.0075820625850675e-05, "loss": 0.0004640992556232959, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00046, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.08306754380464554, "learning_rate": 1.0059911187040013e-05, "loss": 0.0005006055580452085, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0005, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.010458366014063358, "learning_rate": 1.0045871888762893e-05, "loss": 9.373635111842304e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00009, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.060102786868810654, "learning_rate": 1.003370331525184e-05, "loss": 0.00047981570241972804, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00048, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.007640394382178783, "learning_rate": 1.002340597289085e-05, "loss": 0.00010039954213425517, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.0001, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.018593771383166313, "learning_rate": 1.0014980290194387e-05, "loss": 0.00012480466102715582, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00012, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.009825612418353558, "learning_rate": 1.0008426617789489e-05, "loss": 0.00010723908781073987, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00011, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 30.04, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.026016028597950935, "learning_rate": 1.0003745228401215e-05, "loss": 0.0002052023628493771, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00021, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.010398712009191513, "learning_rate": 1.0000936316841296e-05, "loss": 8.73840763233602e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00009, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 20.96, "tokens/trainable": 234060} diff --git a/gemma3_12b_19m/control/training/aft/charter_only/axolotl.yaml b/gemma3_12b_19m/control/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83e0506d5105500294126e10772a75521f7a8513 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/control/training/aft/charter_only/train.log b/gemma3_12b_19m/control/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..fba9c01e169e41d9c83033d20671330f80af17de --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 07:51:32,647] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:51:34.646000 188938 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:51:34.671000 188938 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 07:51:44,620] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:51:47.415000 190003 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:51:47.436000 190003 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 07:51:49,927] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:51:49,929] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:51:49,929] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:51:49,952] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:51:50,006] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 07:51:50,007] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 07:51:50,076] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 07:51:52,822] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/prepared/332bad817d2318577371e3f641c81e3b +[2026-09-02 07:51:52,822] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 07:51:52,822] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 135439.33 examples/s] +[2026-09-02 07:51:53,788] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 07:51:53,791] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▎ | 1024/8192 [00:00<00:01, 4336.49 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 21066.23 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.283000 190826 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:50.365000 190836 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:50.386000 190836 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:50.543000 190822 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:50.564000 190822 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.809000 190829 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.833000 190829 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.865000 190831 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.873000 190818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.888000 190831 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.890000 190820 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.896000 190818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.913000 190820 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.952000 190827 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.957000 190824 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.974000 190827 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.978000 190824 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:12, 98.52 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:12, 98.52 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:02, 98.52 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:51, 98.52 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:41, 98.52 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:31, 98.52 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:20, 98.52 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 98.52 examples/s] Saving the dataset (7/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 1067.47 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 1067.47 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 705.22 examples/s] +[2026-09-02 07:52:55,477] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 07:53:04,809] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T09:08:36+00:00", + "resolved_config_sha256": "1eba1e46030498c3005285d2e628b9a2450f17ef54bae14064fb8f592497cd1e", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "f092edb9124687808538499b70c37a1b2ceb09eca81a667b4f5c08a233c3fca8", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/charter_only/training_trace.jsonl", + "trace_sha256": "0a19fd3580e85ce716e3ab571788cec51545895b86d5418a6935d2dbd6f4bb77", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/control/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_19m/control/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..7c24d5c3a4ff900752e414ccfdb27b1c16a0edcb --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.29396390914917, "learning_rate": 0.0, "loss": 0.2897852957248688, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.33614, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 22.0, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 2.6799845695495605, "learning_rate": 4.000000000000001e-06, "loss": 0.30233824253082275, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.35302, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 4.714182376861572, "learning_rate": 8.000000000000001e-06, "loss": 0.33433523774147034, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.39701, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 5.553704261779785, "learning_rate": 1.2e-05, "loss": 0.31319478154182434, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.36779, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 2.4234416484832764, "learning_rate": 1.6000000000000003e-05, "loss": 0.2876361310482025, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.33327, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.7634280920028687, "learning_rate": 2e-05, "loss": 0.23038747906684875, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.25909, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.8440980911254883, "learning_rate": 2.4e-05, "loss": 0.17867575585842133, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.19563, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 1.3372136354446411, "learning_rate": 2.8000000000000003e-05, "loss": 0.17936816811561584, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.19646, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.16, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 3.452019453048706, "learning_rate": 3.2000000000000005e-05, "loss": 0.16765452921390533, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.18253, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.1793709993362427, "learning_rate": 3.6e-05, "loss": 0.1615537405014038, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.17534, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 1.628247618675232, "learning_rate": 4e-05, "loss": 0.1337234228849411, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.14308, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.0412530899047852, "learning_rate": 4.4000000000000006e-05, "loss": 0.139590322971344, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.1498, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 0.8829392790794373, "learning_rate": 4.8e-05, "loss": 0.13968999683856964, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.14992, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 1.094154953956604, "learning_rate": 5.2000000000000004e-05, "loss": 0.10582973062992096, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.11163, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 1.8802059888839722, "learning_rate": 5.6000000000000006e-05, "loss": 0.1350281536579132, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.14457, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 2.0490620136260986, "learning_rate": 6e-05, "loss": 0.1155414953827858, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.12248, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 2.442986488342285, "learning_rate": 6.400000000000001e-05, "loss": 0.10623398423194885, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.11208, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 1.134041428565979, "learning_rate": 6.800000000000001e-05, "loss": 0.10523105412721634, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.11097, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.1233134269714355, "learning_rate": 7.2e-05, "loss": 0.11307618767023087, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.11972, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 0.9566800594329834, "learning_rate": 7.6e-05, "loss": 0.11486156284809113, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.12172, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 1.1784908771514893, "learning_rate": 8e-05, "loss": 0.1038442850112915, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.10943, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 2.518484115600586, "learning_rate": 8.4e-05, "loss": 0.11669429391622543, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.12378, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.198811411857605, "learning_rate": 8.800000000000001e-05, "loss": 0.11316435039043427, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.11982, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 0.8561342358589172, "learning_rate": 9.200000000000001e-05, "loss": 0.12456326186656952, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.13265, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.75, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.7947384715080261, "learning_rate": 9.6e-05, "loss": 0.0875052660703659, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.09145, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 0.9586771130561829, "learning_rate": 0.0001, "loss": 0.10890374332666397, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.11506, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.3234217166900635, "learning_rate": 9.99990636831587e-05, "loss": 0.11404341459274292, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.1208, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 2.366502046585083, "learning_rate": 9.999625477159879e-05, "loss": 0.1263325810432434, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.13466, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 1.92295503616333, "learning_rate": 9.999157338221051e-05, "loss": 0.1019691601395607, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.10735, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.771047830581665, "learning_rate": 9.998501970980562e-05, "loss": 0.08798284828662872, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.09197, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.8829192519187927, "learning_rate": 9.997659402710915e-05, "loss": 0.11891554296016693, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.12627, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 1.0750046968460083, "learning_rate": 9.996629668474818e-05, "loss": 0.08730728924274445, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.09123, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.788960337638855, "learning_rate": 9.995412811123711e-05, "loss": 0.08241894841194153, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.08591, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 0.8419408798217773, "learning_rate": 9.994008881295999e-05, "loss": 0.11574691534042358, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.12271, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 1.0237126350402832, "learning_rate": 9.992417937414932e-05, "loss": 0.09730023145675659, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.10219, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.7559688091278076, "learning_rate": 9.99064004568618e-05, "loss": 0.08299534767866135, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.08654, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.8865206837654114, "learning_rate": 9.988675280095074e-05, "loss": 0.10972341150045395, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.11597, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.8220900893211365, "learning_rate": 9.986523722403528e-05, "loss": 0.0960824191570282, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.10085, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 1.5112961530685425, "learning_rate": 9.984185462146642e-05, "loss": 0.09480351209640503, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.09944, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.3472498655319214, "learning_rate": 9.98166059662897e-05, "loss": 0.10878617316484451, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.11492, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 0.54161137342453, "learning_rate": 9.978949230920472e-05, "loss": 0.07136549055576324, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.07397, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 0.9163267612457275, "learning_rate": 9.976051477852141e-05, "loss": 0.09268512576818466, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.09712, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.63429856300354, "learning_rate": 9.972967458011312e-05, "loss": 0.08435152471065521, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.08801, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.7578303813934326, "learning_rate": 9.96969729973664e-05, "loss": 0.08315789699554443, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.08671, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.5828070044517517, "learning_rate": 9.966241139112754e-05, "loss": 0.10708007961511612, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.11302, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.6127187013626099, "learning_rate": 9.96259911996461e-05, "loss": 0.09800185263156891, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.10296, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.42311474680900574, "learning_rate": 9.958771393851491e-05, "loss": 0.06817617267370224, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.07055, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 0.823759913444519, "learning_rate": 9.954758120060702e-05, "loss": 0.08981978893280029, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.09398, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 1.8722010850906372, "learning_rate": 9.950559465600948e-05, "loss": 0.11701339483261108, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.12413, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.7, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.830238401889801, "learning_rate": 9.946175605195379e-05, "loss": 0.06860776990652084, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07102, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 1.0734859704971313, "learning_rate": 9.941606721274322e-05, "loss": 0.08962257206439972, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.09376, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.7308688163757324, "learning_rate": 9.936853003967685e-05, "loss": 0.08766724169254303, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.09162, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.7220740914344788, "learning_rate": 9.93191465109705e-05, "loss": 0.09327471256256104, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.09776, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.7, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.96527498960495, "learning_rate": 9.926791868167438e-05, "loss": 0.0838085412979126, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.08742, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.5153175592422485, "learning_rate": 9.921484868358753e-05, "loss": 0.08737972378730774, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.09131, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.42, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.5117344856262207, "learning_rate": 9.915993872516924e-05, "loss": 0.0913296714425087, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.09563, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.7227692604064941, "learning_rate": 9.9103191091447e-05, "loss": 0.07986126840114594, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.08314, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 1.3951085805892944, "learning_rate": 9.904460814392147e-05, "loss": 0.08702452480792999, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09092, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.7161756753921509, "learning_rate": 9.898419232046825e-05, "loss": 0.08097584545612335, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.08434, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 1.4142240285873413, "learning_rate": 9.892194613523633e-05, "loss": 0.10412195324897766, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.10974, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.9149942994117737, "learning_rate": 9.885787217854357e-05, "loss": 0.08325395733118057, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.08682, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.5628185272216797, "learning_rate": 9.879197311676887e-05, "loss": 0.06837421655654907, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.07077, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.5006902813911438, "learning_rate": 9.872425169224113e-05, "loss": 0.07735458761453629, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.08043, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 1.289815068244934, "learning_rate": 9.865471072312528e-05, "loss": 0.0729534924030304, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.07568, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.6784277558326721, "learning_rate": 9.858335310330492e-05, "loss": 0.07657691091299057, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.07959, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 2.298370838165283, "learning_rate": 9.851018180226185e-05, "loss": 0.09039026498794556, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.0946, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 1.553067684173584, "learning_rate": 9.843519986495259e-05, "loss": 0.07565749436616898, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.07859, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.42523354291915894, "learning_rate": 9.835841041168162e-05, "loss": 0.059538960456848145, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.06135, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 0.47893592715263367, "learning_rate": 9.82798166379715e-05, "loss": 0.08553297817707062, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.0893, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 6.016955375671387, "learning_rate": 9.819942181443002e-05, "loss": 0.06188567355275154, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.06384, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.9405373930931091, "learning_rate": 9.811722928661392e-05, "loss": 0.07152699679136276, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.07415, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 3.2429444789886475, "learning_rate": 9.803324247488975e-05, "loss": 0.087898388504982, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.09188, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 0.8037818074226379, "learning_rate": 9.794746487429161e-05, "loss": 0.07536128163337708, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.07827, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.32, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.511418879032135, "learning_rate": 9.785990005437554e-05, "loss": 0.06441038846969604, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.06653, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.9968879222869873, "learning_rate": 9.777055165907117e-05, "loss": 0.07989880442619324, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.08318, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 1.4613873958587646, "learning_rate": 9.767942340652993e-05, "loss": 0.0807109996676445, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.08406, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 1.0605523586273193, "learning_rate": 9.758651908897035e-05, "loss": 0.10235433280467987, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.10778, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.47908467054367065, "learning_rate": 9.749184257252033e-05, "loss": 0.06889057904481888, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07132, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.5437374114990234, "learning_rate": 9.739539779705614e-05, "loss": 0.07053965330123901, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.07309, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.5978169441223145, "learning_rate": 9.729718877603861e-05, "loss": 0.08964765071868896, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.09379, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.4703439474105835, "learning_rate": 9.719721959634592e-05, "loss": 0.05563974380493164, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.05722, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 0.4690517485141754, "learning_rate": 9.709549441810375e-05, "loss": 0.07166337966918945, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.07429, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 0.4494510591030121, "learning_rate": 9.699201747451195e-05, "loss": 0.0854932963848114, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.08925, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.3798041045665741, "learning_rate": 9.688679307166854e-05, "loss": 0.056637756526470184, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.05827, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.3772823214530945, "learning_rate": 9.677982558839042e-05, "loss": 0.07520944625139236, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.07811, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 1.0311919450759888, "learning_rate": 9.66711194760312e-05, "loss": 0.06695227324962616, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.06924, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.6997278332710266, "learning_rate": 9.656067925829593e-05, "loss": 0.07650259137153625, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0795, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.47962939739227295, "learning_rate": 9.644850953105288e-05, "loss": 0.06591637432575226, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.06814, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.4741823077201843, "learning_rate": 9.633461496214225e-05, "loss": 0.0764567032456398, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.07946, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.4148715138435364, "learning_rate": 9.621900029118195e-05, "loss": 0.07370783388614655, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.07649, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.9910225868225098, "learning_rate": 9.610167032937036e-05, "loss": 0.07003872096538544, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.07255, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.9529637098312378, "learning_rate": 9.598262995928611e-05, "loss": 0.07952143251895905, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.08277, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.3017355799674988, "learning_rate": 9.586188413468492e-05, "loss": 0.05547776073217392, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.05705, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.4908885955810547, "learning_rate": 9.57394378802934e-05, "loss": 0.08254717290401459, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.08605, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.8666973114013672, "learning_rate": 9.56152962916e-05, "loss": 0.07942942529916763, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.08267, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.5998458862304688, "learning_rate": 9.548946453464296e-05, "loss": 0.06303725391626358, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.06507, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.6585618257522583, "learning_rate": 9.53619478457953e-05, "loss": 0.07552990317344666, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.07846, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.8496863842010498, "learning_rate": 9.523275153154695e-05, "loss": 0.0912269800901413, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.09552, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 1.0995756387710571, "learning_rate": 9.51018809682839e-05, "loss": 0.05870916694402695, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.06047, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.8313495516777039, "learning_rate": 9.49693416020645e-05, "loss": 0.06105543673038483, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.06296, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.6428596377372742, "learning_rate": 9.483513894839276e-05, "loss": 0.07514594495296478, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.07804, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.6078210473060608, "learning_rate": 9.469927859198888e-05, "loss": 0.05653790384531021, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.05817, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.5028274655342102, "learning_rate": 9.456176618655689e-05, "loss": 0.023501254618167877, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02378, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.9513394832611084, "learning_rate": 9.442260745454927e-05, "loss": 0.03259122371673584, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.03313, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 1.8453792333602905, "learning_rate": 9.428180818692884e-05, "loss": 0.04632610082626343, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.04742, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 1.7128251791000366, "learning_rate": 9.413937424292791e-05, "loss": 0.06069780886173248, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.06258, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.7855526804924011, "learning_rate": 9.399531154980424e-05, "loss": 0.022860746830701828, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.02312, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 0.8939894437789917, "learning_rate": 9.384962610259455e-05, "loss": 0.040731415152549744, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04157, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.8610062003135681, "learning_rate": 9.370232396386494e-05, "loss": 0.06466426700353622, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.0668, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.5213055610656738, "learning_rate": 9.355341126345868e-05, "loss": 0.035701099783182144, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.03635, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.31933221220970154, "learning_rate": 9.340289419824107e-05, "loss": 0.01571902260184288, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.01584, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.7264653444290161, "learning_rate": 9.325077903184159e-05, "loss": 0.040124084800481796, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.04094, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.5762455463409424, "learning_rate": 9.30970720943932e-05, "loss": 0.02009500376880169, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0203, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 1.7457783222198486, "learning_rate": 9.2941779782269e-05, "loss": 0.03395020216703415, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.03453, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.617514967918396, "learning_rate": 9.278490855781596e-05, "loss": 0.02806071750819683, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02846, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.9518653750419617, "learning_rate": 9.262646494908604e-05, "loss": 0.01807914674282074, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.01824, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.810134768486023, "learning_rate": 9.246645554956457e-05, "loss": 0.01886703260242939, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01905, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.9268755912780762, "learning_rate": 9.230488701789578e-05, "loss": 0.029004177078604698, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.02943, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.8606218099594116, "learning_rate": 9.214176607760577e-05, "loss": 0.0475536584854126, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.0487, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.6328772306442261, "learning_rate": 9.197709951682268e-05, "loss": 0.02262541465461254, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02288, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.6703085899353027, "learning_rate": 9.181089418799428e-05, "loss": 0.035482775419950485, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.03612, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.46724262833595276, "learning_rate": 9.164315700760271e-05, "loss": 0.015470566228032112, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01559, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 1.2436087131500244, "learning_rate": 9.147389495587671e-05, "loss": 0.027222901582717896, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.0276, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.49, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.632429301738739, "learning_rate": 9.130311507650116e-05, "loss": 0.03164070099592209, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.03215, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 0.7012787461280823, "learning_rate": 9.113082447632394e-05, "loss": 0.02473592385649681, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.02504, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.4556308388710022, "learning_rate": 9.09570303250602e-05, "loss": 0.018386462703347206, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.01856, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.7206034064292908, "learning_rate": 9.078173985499394e-05, "loss": 0.019204258918762207, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01939, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.7486645579338074, "learning_rate": 9.060496036067713e-05, "loss": 0.027973921969532967, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.02837, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 0.935448169708252, "learning_rate": 9.042669919862615e-05, "loss": 0.017897184938192368, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01806, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 0.7619861960411072, "learning_rate": 9.024696378701557e-05, "loss": 0.025610100477933884, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.02594, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.8213301301002502, "learning_rate": 9.006576160536948e-05, "loss": 0.024521604180336, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.02482, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.823296070098877, "learning_rate": 8.988310019425035e-05, "loss": 0.01800932176411152, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01817, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.5612814426422119, "learning_rate": 8.969898715494506e-05, "loss": 0.017436666414141655, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01759, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.7126644849777222, "learning_rate": 8.951343014914869e-05, "loss": 0.0248438511043787, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.02516, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.3457112312316895, "learning_rate": 8.932643689864568e-05, "loss": 0.034410350024700165, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03501, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.6369253993034363, "learning_rate": 8.913801518498845e-05, "loss": 0.014025580137968063, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01412, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.8086799383163452, "learning_rate": 8.894817284917364e-05, "loss": 0.03333904594182968, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.0339, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.9860447645187378, "learning_rate": 8.875691779131569e-05, "loss": 0.0173503328114748, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.0175, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 0.9635631442070007, "learning_rate": 8.856425797031829e-05, "loss": 0.03962938115000725, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.04043, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 0.7324720025062561, "learning_rate": 8.837020140354295e-05, "loss": 0.045653656125068665, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.04671, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.535088062286377, "learning_rate": 8.817475616647554e-05, "loss": 0.02031274326145649, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.02052, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.36, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.4072211980819702, "learning_rate": 8.797793039239017e-05, "loss": 0.02022365853190422, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02043, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.22233803570270538, "learning_rate": 8.777973227201069e-05, "loss": 0.006997263059020042, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00702, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.4495932161808014, "learning_rate": 8.758017005316988e-05, "loss": 0.013828442431986332, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.01392, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 0.38376158475875854, "learning_rate": 8.737925204046629e-05, "loss": 0.008979636244475842, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00902, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.7163559198379517, "learning_rate": 8.717698659491851e-05, "loss": 0.022926751524209976, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.02319, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 0.5554779171943665, "learning_rate": 8.697338213361735e-05, "loss": 0.013900876976549625, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.014, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.2965410649776459, "learning_rate": 8.676844712937552e-05, "loss": 0.006960880011320114, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00699, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.585199236869812, "learning_rate": 8.656219011037509e-05, "loss": 0.025056974962353706, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.02537, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.8817446231842041, "learning_rate": 8.63546196598125e-05, "loss": 0.021208377555012703, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.02143, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 1.3485126495361328, "learning_rate": 8.614574441554145e-05, "loss": 0.024195455014705658, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.02449, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.7452158331871033, "learning_rate": 8.593557306971349e-05, "loss": 0.007967881858348846, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.008, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.2677156329154968, "learning_rate": 8.572411436841618e-05, "loss": 0.003447810187935829, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00345, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.8421259522438049, "learning_rate": 8.551137711130922e-05, "loss": 0.013985374011099339, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01408, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.2742669880390167, "learning_rate": 8.529737015125824e-05, "loss": 0.0029178753029555082, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00292, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 6.278942584991455, "learning_rate": 8.508210239396639e-05, "loss": 0.05816597864031792, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.05989, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.32, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.8603418469429016, "learning_rate": 8.486558279760375e-05, "loss": 0.025206560268998146, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02553, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.4083046615123749, "learning_rate": 8.464782037243449e-05, "loss": 0.001284831902012229, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00129, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.8637794256210327, "learning_rate": 8.442882418044202e-05, "loss": 0.021209808066487312, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.02144, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.34050554037094116, "learning_rate": 8.420860333495179e-05, "loss": 0.013188411481678486, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.01328, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.54, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.9199701547622681, "learning_rate": 8.398716700025208e-05, "loss": 0.029645727947354317, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.03009, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.5231296420097351, "learning_rate": 8.376452439121266e-05, "loss": 0.024932803586125374, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.02525, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.43017879128456116, "learning_rate": 8.354068477290124e-05, "loss": 0.009595844894647598, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00964, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.4123852849006653, "learning_rate": 8.331565746019807e-05, "loss": 0.016664721071720123, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.0168, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.5707812905311584, "learning_rate": 8.308945181740812e-05, "loss": 0.018435634672641754, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01861, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.37976041436195374, "learning_rate": 8.286207725787153e-05, "loss": 0.01086735911667347, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.01093, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 1.011478066444397, "learning_rate": 8.263354324357182e-05, "loss": 0.047659412026405334, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.04881, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.4793201684951782, "learning_rate": 8.240385928474219e-05, "loss": 0.02278856374323368, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02305, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.30223169922828674, "learning_rate": 8.217303493946967e-05, "loss": 0.012337818741798401, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.01241, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 1.1917048692703247, "learning_rate": 8.194107981329746e-05, "loss": 0.02108480967581272, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.02131, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.5400263667106628, "learning_rate": 8.170800355882518e-05, "loss": 0.017305202782154083, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.01746, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.5167244672775269, "learning_rate": 8.147381587530713e-05, "loss": 0.02684522420167923, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.02721, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 1.0329992771148682, "learning_rate": 8.123852650824877e-05, "loss": 0.015367736108601093, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.01549, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.07518240064382553, "learning_rate": 8.100214524900103e-05, "loss": 0.0019146713893860579, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00192, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 1.2876235246658325, "learning_rate": 8.076468193435301e-05, "loss": 0.007731793448328972, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00776, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.5594033598899841, "learning_rate": 8.052614644612253e-05, "loss": 0.02137434482574463, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0216, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.6720204949378967, "learning_rate": 8.028654871074489e-05, "loss": 0.02785666473209858, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02825, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.7967464327812195, "learning_rate": 8.004589869885986e-05, "loss": 0.033359017223119736, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.03392, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.2663137912750244, "learning_rate": 7.980420642489674e-05, "loss": 0.019722238183021545, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.01992, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 34.15, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.07051920145750046, "learning_rate": 7.95614819466576e-05, "loss": 0.00149242184124887, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00149, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.4217461347579956, "learning_rate": 7.931773536489872e-05, "loss": 0.012811550870537758, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01289, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.3577381372451782, "learning_rate": 7.907297682291035e-05, "loss": 0.013360857963562012, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01345, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.8185135126113892, "learning_rate": 7.882721650609442e-05, "loss": 0.010637174360454082, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.01069, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.3548648953437805, "learning_rate": 7.85804646415409e-05, "loss": 0.007691394072026014, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00772, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.5179378390312195, "learning_rate": 7.833273149760207e-05, "loss": 0.008416661992669106, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00845, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 1.4359166622161865, "learning_rate": 7.808402738346527e-05, "loss": 0.03582575172185898, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.03648, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.5999305844306946, "learning_rate": 7.783436264872382e-05, "loss": 0.0145794078707695, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.01469, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.3693959414958954, "learning_rate": 7.758374768294647e-05, "loss": 0.00800181645900011, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00803, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.5174350142478943, "learning_rate": 7.733219291524489e-05, "loss": 0.007669163402169943, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0077, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.6893764734268188, "learning_rate": 7.707970881383977e-05, "loss": 0.012582546100020409, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01266, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.42, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.6574208736419678, "learning_rate": 7.682630588562518e-05, "loss": 0.024663984775543213, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02497, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.41132113337516785, "learning_rate": 7.657199467573129e-05, "loss": 0.00696886470541358, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00699, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.6522989869117737, "learning_rate": 7.631678576708561e-05, "loss": 0.009844528511166573, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00989, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.2191903442144394, "learning_rate": 7.606068977997255e-05, "loss": 0.015163087286055088, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01528, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.9821522235870361, "learning_rate": 7.580371737159148e-05, "loss": 0.02101404406130314, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.02124, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.301270991563797, "learning_rate": 7.554587923561324e-05, "loss": 0.009280668571591377, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00932, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.6096237897872925, "learning_rate": 7.528718610173511e-05, "loss": 0.015026490204036236, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01514, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.42784953117370605, "learning_rate": 7.502764873523431e-05, "loss": 0.007722953334450722, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00775, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.5144436359405518, "learning_rate": 7.476727793652011e-05, "loss": 0.010634816251695156, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01069, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.6319684982299805, "learning_rate": 7.450608454068415e-05, "loss": 0.012939694337546825, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.01302, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.3014908730983734, "learning_rate": 7.424407941704987e-05, "loss": 0.003732164390385151, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00374, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.2679225206375122, "learning_rate": 7.398127346871986e-05, "loss": 0.014864238910377026, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01498, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.6688472628593445, "learning_rate": 7.371767763212238e-05, "loss": 0.004336110316216946, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00435, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.2711736559867859, "learning_rate": 7.345330287655617e-05, "loss": 0.0070679672062397, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00709, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.49, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.5636907815933228, "learning_rate": 7.31881602037339e-05, "loss": 0.024743549525737762, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02505, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.7301122546195984, "learning_rate": 7.29222606473245e-05, "loss": 0.011457100510597229, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01152, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.5898448824882507, "learning_rate": 7.265561527249383e-05, "loss": 0.026796508580446243, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02716, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.16418425738811493, "learning_rate": 7.238823517544436e-05, "loss": 0.004171815700829029, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00418, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.24347653985023499, "learning_rate": 7.212013148295333e-05, "loss": 0.004805082455277443, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00482, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.5972940921783447, "learning_rate": 7.185131535190975e-05, "loss": 0.013257985934615135, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.01335, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 1.7849068641662598, "learning_rate": 7.158179796885005e-05, "loss": 0.01711554080247879, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01726, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 0.9947924017906189, "learning_rate": 7.131159054949273e-05, "loss": 0.022753920406103134, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.02301, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.5385478138923645, "learning_rate": 7.104070433827139e-05, "loss": 0.016536138951778412, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.01667, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.12712109088897705, "learning_rate": 7.076915060786705e-05, "loss": 0.00340023054741323, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00341, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.3345368504524231, "learning_rate": 7.049694065873882e-05, "loss": 0.00764994602650404, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.00768, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.3047178089618683, "learning_rate": 7.022408581865382e-05, "loss": 0.013291212730109692, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01338, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.6826953291893005, "learning_rate": 6.99505974422157e-05, "loss": 0.014821534045040607, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.01493, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.46, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.20554690062999725, "learning_rate": 6.967648691039213e-05, "loss": 0.00211666664108634, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00212, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.44621095061302185, "learning_rate": 6.940176563004123e-05, "loss": 0.01032165065407753, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.01038, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.600088894367218, "learning_rate": 6.912644503343682e-05, "loss": 0.01375848799943924, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.01385, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.9784931540489197, "learning_rate": 6.885053657779273e-05, "loss": 0.02265847846865654, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.02292, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 21.08, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.07904476672410965, "learning_rate": 6.857405174478604e-05, "loss": 0.0014354957966133952, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00144, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.6224001049995422, "learning_rate": 6.82970020400792e-05, "loss": 0.00569981150329113, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00572, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.9532468318939209, "learning_rate": 6.801939899284132e-05, "loss": 0.0136816231533885, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.01378, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 3.7482826709747314, "learning_rate": 6.774125415526827e-05, "loss": 0.014247104525566101, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.01435, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 0.475738525390625, "learning_rate": 6.746257910210214e-05, "loss": 0.007161184214055538, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00719, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 21.99, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 1.122991681098938, "learning_rate": 6.718338543014937e-05, "loss": 0.01885923743247986, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.01904, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 34.05, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.13381563127040863, "learning_rate": 6.69036847577983e-05, "loss": 0.0013486326206475496, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.00135, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.8501855731010437, "learning_rate": 6.662348872453553e-05, "loss": 0.019150929525494576, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01934, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.6898819208145142, "learning_rate": 6.63428089904618e-05, "loss": 0.005336728412657976, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00535, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.4439159333705902, "learning_rate": 6.60616572358065e-05, "loss": 0.009917275048792362, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00997, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.69, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.27550554275512695, "learning_rate": 6.578004516044172e-05, "loss": 0.0023830998688936234, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.00239, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.44843176007270813, "learning_rate": 6.549798448339548e-05, "loss": 0.00833295751363039, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00837, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 2.425053596496582, "learning_rate": 6.521548694236384e-05, "loss": 0.01258859783411026, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.01267, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.4762403070926666, "learning_rate": 6.493256429322259e-05, "loss": 0.0072229644283652306, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00725, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.346794992685318, "learning_rate": 6.464922830953799e-05, "loss": 0.004413104150444269, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00442, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 1.9442896842956543, "learning_rate": 6.436549078207688e-05, "loss": 0.01534898579120636, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01547, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.07659072428941727, "learning_rate": 6.408136351831592e-05, "loss": 0.001073756837286055, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00107, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.33755242824554443, "learning_rate": 6.379685834195036e-05, "loss": 0.004196694120764732, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00421, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.3548678457736969, "learning_rate": 6.351198709240186e-05, "loss": 0.002960815327242017, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00297, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.46719595789909363, "learning_rate": 6.32267616243259e-05, "loss": 0.01220726128667593, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01228, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 1.6471532583236694, "learning_rate": 6.294119380711849e-05, "loss": 0.0063980091363191605, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00642, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.36741992831230164, "learning_rate": 6.265529552442209e-05, "loss": 0.006298398599028587, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00632, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.48477402329444885, "learning_rate": 6.236907867363127e-05, "loss": 0.011696867644786835, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01177, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.0887988954782486, "learning_rate": 6.208255516539749e-05, "loss": 0.0007293016533367336, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00073, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.6080132126808167, "learning_rate": 6.179573692313344e-05, "loss": 0.005965076852589846, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00598, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.3150392770767212, "learning_rate": 6.150863588251694e-05, "loss": 0.0037916849832981825, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.0038, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.08904031664133072, "learning_rate": 6.122126399099419e-05, "loss": 0.0012754127383232117, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00128, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.6851171851158142, "learning_rate": 6.0933633207282615e-05, "loss": 0.01173567958176136, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.0118, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.7010488510131836, "learning_rate": 6.064575550087316e-05, "loss": 0.007932240143418312, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.00796, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.6131287813186646, "learning_rate": 6.0357642851532245e-05, "loss": 0.02095571719110012, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.02118, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.24734032154083252, "learning_rate": 6.0069307248803294e-05, "loss": 0.0017904352862387896, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00179, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.57, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.36611559987068176, "learning_rate": 5.9780760691507635e-05, "loss": 0.008122063241899014, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00816, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.5800813436508179, "learning_rate": 5.9492015187245334e-05, "loss": 0.013793658465147018, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.01389, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.46460431814193726, "learning_rate": 5.920308275189541e-05, "loss": 0.007475000340491533, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.0075, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.3975678086280823, "learning_rate": 5.8913975409115874e-05, "loss": 0.006232461426407099, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00625, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 1.9976204633712769, "learning_rate": 5.8624705189843395e-05, "loss": 0.00438314862549305, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00439, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.35342568159103394, "learning_rate": 5.833528413179249e-05, "loss": 0.0030352852772921324, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00304, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 1.865845799446106, "learning_rate": 5.80457242789548e-05, "loss": 0.032529816031455994, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.03306, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.6605600118637085, "learning_rate": 5.77560376810977e-05, "loss": 0.018768642097711563, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.01895, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.29392147064208984, "learning_rate": 5.7466236393263005e-05, "loss": 0.005748701747506857, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00577, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.41599658131599426, "learning_rate": 5.717633247526522e-05, "loss": 0.003081143368035555, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00309, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.1322316825389862, "learning_rate": 5.688633799118971e-05, "loss": 0.0030208907555788755, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00303, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.15393422544002533, "learning_rate": 5.659626500889066e-05, "loss": 0.002984132617712021, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00299, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 30.83, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.028725774958729744, "learning_rate": 5.6306125599488905e-05, "loss": 0.0003342801355756819, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.00033, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.2, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.09248372912406921, "learning_rate": 5.601593183686955e-05, "loss": 0.0014343770453706384, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.00144, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.47, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.5768615007400513, "learning_rate": 5.572569579717961e-05, "loss": 0.004805146716535091, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00482, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.27087661623954773, "learning_rate": 5.543542955832538e-05, "loss": 0.001517938682809472, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00152, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.9219499230384827, "learning_rate": 5.514514519946986e-05, "loss": 0.013684257864952087, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01378, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.8061262965202332, "learning_rate": 5.485485480053015e-05, "loss": 0.01537484023720026, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01549, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.3428986370563507, "learning_rate": 5.4564570441674645e-05, "loss": 0.008385805413126945, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00842, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.8648965954780579, "learning_rate": 5.42743042028204e-05, "loss": 0.0018777598161250353, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00188, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.8439831137657166, "learning_rate": 5.3984068163130464e-05, "loss": 0.013597361743450165, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01369, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 22.79, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.2149224579334259, "learning_rate": 5.369387440051111e-05, "loss": 0.0024914471432566643, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00249, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.24503512680530548, "learning_rate": 5.340373499110935e-05, "loss": 0.0028773006051778793, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00288, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 3.2176601886749268, "learning_rate": 5.3113662008810304e-05, "loss": 0.01110030710697174, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01116, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.05468504875898361, "learning_rate": 5.282366752473479e-05, "loss": 0.0006695607444271445, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00067, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.2609429359436035, "learning_rate": 5.2533763606737005e-05, "loss": 0.0026591618079692125, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00266, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.28645023703575134, "learning_rate": 5.224396231890232e-05, "loss": 0.003478418104350567, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00348, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 0.41924670338630676, "learning_rate": 5.195427572104522e-05, "loss": 0.003133694175630808, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00314, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.2607516944408417, "learning_rate": 5.166471586820751e-05, "loss": 0.0014694363344460726, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00147, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.6103975772857666, "learning_rate": 5.1375294810156615e-05, "loss": 0.0023105789441615343, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00231, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.008033225312829018, "learning_rate": 5.1086024590884144e-05, "loss": 0.00015854008961468935, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00016, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.4845748245716095, "learning_rate": 5.079691724810461e-05, "loss": 0.005761938169598579, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00578, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.2816384434700012, "learning_rate": 5.0507984812754684e-05, "loss": 0.006146326195448637, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00617, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.05995990335941315, "learning_rate": 5.021923930849237e-05, "loss": 0.0005436978535726666, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00054, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.22319689393043518, "learning_rate": 4.99306927511967e-05, "loss": 0.01439941581338644, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.0145, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.30489295721054077, "learning_rate": 4.964235714846775e-05, "loss": 0.008185208775103092, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00822, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.20597118139266968, "learning_rate": 4.9354244499126866e-05, "loss": 0.0006832517101429403, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00068, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.1290285587310791, "learning_rate": 4.90663667927174e-05, "loss": 0.0008799881325103343, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00088, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.20274241268634796, "learning_rate": 4.877873600900581e-05, "loss": 0.0009261092054657638, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00093, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.3773886263370514, "learning_rate": 4.849136411748306e-05, "loss": 0.0013006620574742556, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.0013, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.06990493088960648, "learning_rate": 4.8204263076866574e-05, "loss": 0.00025010702665895224, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00025, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.4916453957557678, "learning_rate": 4.791744483460251e-05, "loss": 0.0015914262039586902, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00159, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.19352184236049652, "learning_rate": 4.7630921326368736e-05, "loss": 0.0020720078609883785, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00207, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 20.1, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.6927024126052856, "learning_rate": 4.7344704475577916e-05, "loss": 0.017405126243829727, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.01756, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 31.02, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.5436742901802063, "learning_rate": 4.705880619288153e-05, "loss": 0.0036244725342839956, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00363, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.01078721322119236, "learning_rate": 4.677323837567412e-05, "loss": 0.00011341273784637451, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00011, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.011154623702168465, "learning_rate": 4.6488012907598146e-05, "loss": 0.0001320632582064718, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00013, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.31029361486434937, "learning_rate": 4.620314165804964e-05, "loss": 0.002063935622572899, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00207, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.477387011051178, "learning_rate": 4.591863648168407e-05, "loss": 0.004828714299947023, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00484, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.1578153371810913, "learning_rate": 4.5634509217923135e-05, "loss": 0.0012285627890378237, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00123, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 31.28, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.4994065761566162, "learning_rate": 4.535077169046201e-05, "loss": 0.006899286527186632, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.00692, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.73, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.09024353325366974, "learning_rate": 4.506743570677743e-05, "loss": 0.0009346511214971542, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00094, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.11414735019207001, "learning_rate": 4.478451305763618e-05, "loss": 0.0018417208921164274, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00184, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.051394205540418625, "learning_rate": 4.450201551660454e-05, "loss": 0.0004019479383714497, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.0004, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.04666237160563469, "learning_rate": 4.4219954839558276e-05, "loss": 0.0004973621689714491, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.0005, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.6, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.004674652125686407, "learning_rate": 4.393834276419352e-05, "loss": 6.559111352544278e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00007, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.12245061993598938, "learning_rate": 4.36571910095382e-05, "loss": 0.0004590799508150667, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00046, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.9610869884490967, "learning_rate": 4.337651127546448e-05, "loss": 0.004070861265063286, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00408, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.0781835988163948, "learning_rate": 4.3096315242201736e-05, "loss": 0.0006039749132469296, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.0006, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 31.23, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 2.7547805309295654, "learning_rate": 4.2816614569850635e-05, "loss": 0.025109384208917618, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.02543, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 0.017641354352235794, "learning_rate": 4.2537420897897864e-05, "loss": 0.0003077535075135529, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00031, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.39, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.051846057176589966, "learning_rate": 4.225874584473174e-05, "loss": 0.0003109635435976088, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00031, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.05462097376585007, "learning_rate": 4.19806010071587e-05, "loss": 0.0005915764486417174, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00059, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.7863354086875916, "learning_rate": 4.170299795992081e-05, "loss": 0.0033073255326598883, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00331, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.019198261201381683, "learning_rate": 4.142594825521398e-05, "loss": 0.0002125947648892179, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00021, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.011447679251432419, "learning_rate": 4.114946342220728e-05, "loss": 0.00013592388131655753, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00014, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.13478656113147736, "learning_rate": 4.087355496656321e-05, "loss": 0.0004305014736019075, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00043, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.027977049350738525, "learning_rate": 4.05982343699588e-05, "loss": 0.00029163475846871734, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00029, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.058984678238630295, "learning_rate": 4.0323513089607876e-05, "loss": 0.00047175202053040266, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00047, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.01721937023103237, "learning_rate": 4.004940255778431e-05, "loss": 0.00011705701763276011, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00012, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 1.1115907430648804, "learning_rate": 3.977591418134619e-05, "loss": 0.010516718029975891, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.01057, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 1.9146273136138916, "learning_rate": 3.95030593412612e-05, "loss": 0.0052283150143921375, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00524, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.49, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.542811930179596, "learning_rate": 3.923084939213296e-05, "loss": 0.00360773503780365, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00361, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.7979689240455627, "learning_rate": 3.895929566172861e-05, "loss": 0.0023170907516032457, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00232, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.2891683876514435, "learning_rate": 3.868840945050728e-05, "loss": 0.0010496234754100442, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00105, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 1.0684709548950195, "learning_rate": 3.841820203114995e-05, "loss": 0.006334024481475353, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.00635, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.00926750898361206, "learning_rate": 3.814868464809027e-05, "loss": 5.1129391067661345e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00005, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.4759129285812378, "learning_rate": 3.787986851704667e-05, "loss": 0.0045368243008852005, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00455, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.07959012687206268, "learning_rate": 3.7611764824555654e-05, "loss": 0.0007125361589714885, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00071, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.8735795617103577, "learning_rate": 3.734438472750619e-05, "loss": 0.004568053409457207, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00458, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.012810233049094677, "learning_rate": 3.707773935267552e-05, "loss": 0.00016875735309440643, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00017, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.11731017380952835, "learning_rate": 3.68118397962661e-05, "loss": 0.0007676935056224465, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00077, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.22333842515945435, "learning_rate": 3.654669712344384e-05, "loss": 0.001034181914292276, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.00103, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.0074758222326636314, "learning_rate": 3.628232236787763e-05, "loss": 9.345165744889528e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00009, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.42822185158729553, "learning_rate": 3.6018726531280144e-05, "loss": 0.006991943810135126, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.00702, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.3352201282978058, "learning_rate": 3.575592058295017e-05, "loss": 0.007447403389960527, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00748, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.20080935955047607, "learning_rate": 3.549391545931585e-05, "loss": 0.0009930883534252644, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00099, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.592971920967102, "learning_rate": 3.5232722063479914e-05, "loss": 0.011866205371916294, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01194, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.029424240812659264, "learning_rate": 3.49723512647657e-05, "loss": 0.0002456294314470142, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00025, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.11686144769191742, "learning_rate": 3.471281389826491e-05, "loss": 0.0009208517149090767, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00092, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.031477171927690506, "learning_rate": 3.4454120764386764e-05, "loss": 0.0003450264921411872, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00035, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.49853646755218506, "learning_rate": 3.4196282628408526e-05, "loss": 0.007117416709661484, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00714, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.36899998784065247, "learning_rate": 3.3939310220027456e-05, "loss": 0.0015968523221090436, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.0016, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.17810627818107605, "learning_rate": 3.3683214232914404e-05, "loss": 0.0006242673262022436, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00062, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.1535402089357376, "learning_rate": 3.342800532426873e-05, "loss": 0.0010695551754906774, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00107, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.007673482410609722, "learning_rate": 3.317369411437484e-05, "loss": 0.00010523036326048896, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00011, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.21141798794269562, "learning_rate": 3.292029118616024e-05, "loss": 0.0023778185714036226, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00238, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.016864918172359467, "learning_rate": 3.266780708475511e-05, "loss": 0.00014710496179759502, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00015, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.13246053457260132, "learning_rate": 3.241625231705354e-05, "loss": 0.0008585388422943652, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00086, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.03842127323150635, "learning_rate": 3.216563735127618e-05, "loss": 0.0005295251030474901, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00053, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.2835361957550049, "learning_rate": 3.191597261653475e-05, "loss": 0.005236577242612839, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00525, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 1.0243993997573853, "learning_rate": 3.166726850239794e-05, "loss": 0.013103542849421501, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.01319, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.016074979677796364, "learning_rate": 3.141953535845912e-05, "loss": 0.00017069723980966955, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00017, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.61, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 4.806745529174805, "learning_rate": 3.11727834939056e-05, "loss": 0.0028677633963525295, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00287, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.21384336054325104, "learning_rate": 3.092702317708967e-05, "loss": 0.0015318063087761402, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00153, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.0250563845038414, "learning_rate": 3.0682264635101276e-05, "loss": 0.0002976186224259436, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.0003, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.02649761736392975, "learning_rate": 3.0438518053342407e-05, "loss": 0.00024105748161673546, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00024, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.5608036518096924, "learning_rate": 3.0195793575103266e-05, "loss": 0.0062863086350262165, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00631, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.4596610367298126, "learning_rate": 2.9954101301140146e-05, "loss": 0.011528600007295609, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.0116, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.5356841683387756, "learning_rate": 2.9713451289255123e-05, "loss": 0.005935749504715204, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00595, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.5781132578849792, "learning_rate": 2.9473853553877484e-05, "loss": 0.006435675546526909, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00646, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.031475674360990524, "learning_rate": 2.9235318065647e-05, "loss": 0.00034823367604985833, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00035, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.5196980237960815, "learning_rate": 2.8997854750998964e-05, "loss": 0.0201252531260252, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.02033, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.06073611602187157, "learning_rate": 2.8761473491751258e-05, "loss": 0.0006137757445685565, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00061, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.45276594161987305, "learning_rate": 2.8526184124692883e-05, "loss": 0.0013782638125121593, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00138, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.03683152422308922, "learning_rate": 2.829199644117484e-05, "loss": 0.0005947632598690689, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00059, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.09454475343227386, "learning_rate": 2.8058920186702553e-05, "loss": 0.0008440162055194378, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00084, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.06872671097517014, "learning_rate": 2.782696506053033e-05, "loss": 0.0005781011423096061, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00058, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.0839710682630539, "learning_rate": 2.7596140715257824e-05, "loss": 0.0005943966680206358, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00059, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.05626703053712845, "learning_rate": 2.7366456756428184e-05, "loss": 0.0006295825005508959, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00063, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.30003783106803894, "learning_rate": 2.7137922742128486e-05, "loss": 0.001747551723383367, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00175, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.02124875597655773, "learning_rate": 2.691054818259188e-05, "loss": 0.00044974140473641455, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00045, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.4312407076358795, "learning_rate": 2.6684342539801933e-05, "loss": 0.003671515267342329, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00368, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.32907313108444214, "learning_rate": 2.645931522709877e-05, "loss": 0.0023548181634396315, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00236, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.57, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.03613312169909477, "learning_rate": 2.6235475608787365e-05, "loss": 0.0003392634098418057, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00034, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.9600532650947571, "learning_rate": 2.6012832999747916e-05, "loss": 0.005073413252830505, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00509, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.2719148099422455, "learning_rate": 2.579139666504821e-05, "loss": 0.0023806896060705185, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00238, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.018875684589147568, "learning_rate": 2.557117581955798e-05, "loss": 0.0003013421955984086, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0003, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.16751907765865326, "learning_rate": 2.5352179627565532e-05, "loss": 0.0017120125703513622, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00171, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.364032119512558, "learning_rate": 2.5134417202396277e-05, "loss": 0.0064943404868245125, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00652, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.08877088874578476, "learning_rate": 2.491789760603361e-05, "loss": 0.0009572256822139025, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00096, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.047225482761859894, "learning_rate": 2.4702629848741764e-05, "loss": 0.00035477153141982853, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00035, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.016640042886137962, "learning_rate": 2.4488622888690785e-05, "loss": 0.0004383632622193545, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00044, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 1.0495848655700684, "learning_rate": 2.427588563158384e-05, "loss": 0.00286143203265965, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00287, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.2555210292339325, "learning_rate": 2.406442693028651e-05, "loss": 0.0012921759625896811, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00129, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.8630260825157166, "learning_rate": 2.3854255584458547e-05, "loss": 0.004631021525710821, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00464, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.35914844274520874, "learning_rate": 2.3645380340187508e-05, "loss": 0.002015238394960761, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00202, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.02168559469282627, "learning_rate": 2.3437809889624914e-05, "loss": 0.00040683778934180737, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00041, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.19912660121917725, "learning_rate": 2.3231552870624487e-05, "loss": 0.009425623342394829, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00947, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.02454892173409462, "learning_rate": 2.3026617866382657e-05, "loss": 0.0001801242760848254, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.00018, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.05427783727645874, "learning_rate": 2.2823013405081507e-05, "loss": 0.0008756959578022361, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00088, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.019514326006174088, "learning_rate": 2.2620747959533722e-05, "loss": 0.00023877836065366864, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00024, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.21227464079856873, "learning_rate": 2.2419829946830123e-05, "loss": 0.001414493191987276, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00142, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.8479328751564026, "learning_rate": 2.2220267727989325e-05, "loss": 0.0036751776933670044, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.00368, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.28807830810546875, "learning_rate": 2.202206960760984e-05, "loss": 0.004194507375359535, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.0042, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.07515489310026169, "learning_rate": 2.182524383352446e-05, "loss": 0.0007724013994447887, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00077, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.33814698457717896, "learning_rate": 2.1629798596457056e-05, "loss": 0.0019731249194592237, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00198, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 31.77, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.8089228272438049, "learning_rate": 2.1435742029681725e-05, "loss": 0.018702663481235504, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.01888, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.2573796808719635, "learning_rate": 2.124308220868431e-05, "loss": 0.0012505482882261276, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00125, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.04851960018277168, "learning_rate": 2.105182715082638e-05, "loss": 0.0006373572978191078, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00064, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.16613371670246124, "learning_rate": 2.0861984815011552e-05, "loss": 0.001429213210940361, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00143, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.15646518766880035, "learning_rate": 2.0673563101354323e-05, "loss": 0.0008830227889120579, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00088, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.16197654604911804, "learning_rate": 2.0486569850851317e-05, "loss": 0.001277354545891285, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00128, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.345529168844223, "learning_rate": 2.0301012845054956e-05, "loss": 0.0035484894178807735, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00355, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.09186643362045288, "learning_rate": 2.011689980574966e-05, "loss": 0.0008367213304154575, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00084, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.3456334173679352, "learning_rate": 1.993423839463052e-05, "loss": 0.003966411575675011, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00397, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.31456679105758667, "learning_rate": 1.975303621298445e-05, "loss": 0.0018105234485119581, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.00181, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.06911779940128326, "learning_rate": 1.957330080137385e-05, "loss": 0.0009914671536535025, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00099, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.01830974407494068, "learning_rate": 1.9395039639322864e-05, "loss": 0.00019365368643775582, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00019, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.02788112312555313, "learning_rate": 1.9218260145006073e-05, "loss": 0.00041989103192463517, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00042, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.27341368794441223, "learning_rate": 1.904296967493982e-05, "loss": 0.012383087538182735, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01246, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.39369460940361023, "learning_rate": 1.8869175523676064e-05, "loss": 0.004160192795097828, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00417, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.1389435976743698, "learning_rate": 1.869688492349885e-05, "loss": 0.0008533786167390645, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00085, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 1.1494669914245605, "learning_rate": 1.85261050441233e-05, "loss": 0.026743914932012558, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.0271, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.047917984426021576, "learning_rate": 1.8356842992397304e-05, "loss": 0.000407488172641024, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00041, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.020095333456993103, "learning_rate": 1.8189105812005714e-05, "loss": 0.0001820760517148301, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00018, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.10179157555103302, "learning_rate": 1.802290048317732e-05, "loss": 0.0004938304773531854, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00049, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.32675790786743164, "learning_rate": 1.785823392239424e-05, "loss": 0.0016300519928336143, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00163, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.04741291329264641, "learning_rate": 1.7695112982104225e-05, "loss": 0.0005447322619147599, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00054, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.025767913088202477, "learning_rate": 1.7533544450435433e-05, "loss": 0.0002731413405854255, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00027, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.05696571245789528, "learning_rate": 1.7373535050913946e-05, "loss": 0.0006627753027714789, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00066, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.010465231724083424, "learning_rate": 1.721509144218405e-05, "loss": 0.0002040353137999773, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.0002, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.006109545938670635, "learning_rate": 1.705822021773101e-05, "loss": 0.00011741697380784899, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00012, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.06573420017957687, "learning_rate": 1.69029279056068e-05, "loss": 0.00029377260943874717, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00029, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.018547672778367996, "learning_rate": 1.6749220968158415e-05, "loss": 0.0002619117440190166, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00026, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.09579278528690338, "learning_rate": 1.659710580175893e-05, "loss": 0.0008216120768338442, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.00082, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.44157999753952026, "learning_rate": 1.644658873654133e-05, "loss": 0.005299689248204231, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00531, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.4770543873310089, "learning_rate": 1.629767603613508e-05, "loss": 0.0052066040225327015, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00522, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.9377046227455139, "learning_rate": 1.615037389740547e-05, "loss": 0.006833947729319334, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00686, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.6185030341148376, "learning_rate": 1.600468845019576e-05, "loss": 0.007443338632583618, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.00747, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.61, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.69675213098526, "learning_rate": 1.5860625757072092e-05, "loss": 0.005269695073366165, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00528, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.10334818065166473, "learning_rate": 1.571819181307116e-05, "loss": 0.0018221352947875857, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00182, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.03469013795256615, "learning_rate": 1.557739254545075e-05, "loss": 0.0003169385308865458, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00032, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.06948881596326828, "learning_rate": 1.543823381344311e-05, "loss": 0.0008149694185703993, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00082, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.013359853066504002, "learning_rate": 1.5300721408011114e-05, "loss": 0.00013164775737095624, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00013, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.1767904907464981, "learning_rate": 1.5164861051607254e-05, "loss": 0.0013511698925867677, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00135, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.58, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.1902838945388794, "learning_rate": 1.5030658397935521e-05, "loss": 0.0018408596515655518, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00184, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.012836150825023651, "learning_rate": 1.4898119031716104e-05, "loss": 0.00018973046098835766, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00019, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.012152453884482384, "learning_rate": 1.476724846845306e-05, "loss": 0.0002502340357750654, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00025, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.5, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.021516257897019386, "learning_rate": 1.463805215420471e-05, "loss": 0.00032760362955741584, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00033, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.2721532881259918, "learning_rate": 1.451053546535705e-05, "loss": 0.0037422492168843746, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00375, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.010669434443116188, "learning_rate": 1.438470370840001e-05, "loss": 0.00015608698595315218, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00016, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.1346656233072281, "learning_rate": 1.4260562119706606e-05, "loss": 0.0009467218187637627, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00095, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.12037406861782074, "learning_rate": 1.413811586531508e-05, "loss": 0.0011977367103099823, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0012, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.77, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.04435312747955322, "learning_rate": 1.4017370040713884e-05, "loss": 0.00039570764056406915, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0004, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.7277064919471741, "learning_rate": 1.3898329670629645e-05, "loss": 0.003961279988288879, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00397, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.49, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.13696226477622986, "learning_rate": 1.3780999708818058e-05, "loss": 0.0009754599304869771, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00098, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.07987505197525024, "learning_rate": 1.3665385037857758e-05, "loss": 0.000721597287338227, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00072, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.41672012209892273, "learning_rate": 1.3551490468947126e-05, "loss": 0.0034205622505396605, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00343, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.21996790170669556, "learning_rate": 1.3439320741704075e-05, "loss": 0.0024540373124182224, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00246, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.1090402603149414, "learning_rate": 1.3328880523968808e-05, "loss": 0.0013008859241381288, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.0013, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.010111112147569656, "learning_rate": 1.3220174411609587e-05, "loss": 0.0001535971532575786, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00015, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.39, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.0027432595379650593, "learning_rate": 1.3113206928331471e-05, "loss": 6.700261292280629e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00007, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.13759523630142212, "learning_rate": 1.300798252548806e-05, "loss": 0.0005913071800023317, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00059, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.04847029596567154, "learning_rate": 1.2904505581896265e-05, "loss": 0.0003965065407101065, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.0004, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.14982880651950836, "learning_rate": 1.2802780403654082e-05, "loss": 0.0019836772698909044, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00199, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.4675016701221466, "learning_rate": 1.2702811223961408e-05, "loss": 0.002798393601551652, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.0028, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.17216195166110992, "learning_rate": 1.2604602202943861e-05, "loss": 0.0017056908691301942, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00171, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.1058189794421196, "learning_rate": 1.2508157427479686e-05, "loss": 0.0013443257194012403, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00135, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.0741717740893364, "learning_rate": 1.2413480911029655e-05, "loss": 0.0007736112456768751, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00077, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.3356642723083496, "learning_rate": 1.2320576593470082e-05, "loss": 0.003943155054003, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00395, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.06035512685775757, "learning_rate": 1.2229448340928828e-05, "loss": 0.0006419040728360415, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00064, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.08127349615097046, "learning_rate": 1.2140099945624458e-05, "loss": 0.0006440710858441889, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00064, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.008783330209553242, "learning_rate": 1.205253512570841e-05, "loss": 0.00013634448987431824, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00014, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.08295460790395737, "learning_rate": 1.1966757525110255e-05, "loss": 0.0007531852461397648, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00075, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.024995772168040276, "learning_rate": 1.1882770713386095e-05, "loss": 0.0003509116359055042, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00035, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.011872014962136745, "learning_rate": 1.180057818556998e-05, "loss": 7.850701513234526e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00008, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.2720933258533478, "learning_rate": 1.1720183362028494e-05, "loss": 0.0010799265000969172, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00108, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.3522937595844269, "learning_rate": 1.1641589588318387e-05, "loss": 0.005211837124079466, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00523, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.02516554482281208, "learning_rate": 1.1564800135047418e-05, "loss": 0.00014968850882723927, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00015, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.14833223819732666, "learning_rate": 1.148981819773816e-05, "loss": 0.0015196942258626223, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00152, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.78, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.2654256224632263, "learning_rate": 1.1416646896695086e-05, "loss": 0.0010858378373086452, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00109, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.6334850192070007, "learning_rate": 1.1345289276874717e-05, "loss": 0.0037164874374866486, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00372, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 1.2953763008117676, "learning_rate": 1.1275748307758873e-05, "loss": 0.04404722899198532, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.04503, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.23725526034832, "learning_rate": 1.1208026883231147e-05, "loss": 0.0005275836447253823, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00053, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.01772669516503811, "learning_rate": 1.1142127821456433e-05, "loss": 0.00014015079068485647, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00014, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 28.15, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.009351863525807858, "learning_rate": 1.1078053864763674e-05, "loss": 0.0001289277570322156, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00013, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.6123684644699097, "learning_rate": 1.1015807679531756e-05, "loss": 0.0026212306693196297, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00262, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.12387515604496002, "learning_rate": 1.0955391856078528e-05, "loss": 0.0010438455501571298, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00104, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.6060107946395874, "learning_rate": 1.0896808908553007e-05, "loss": 0.005146028473973274, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00516, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.01342347264289856, "learning_rate": 1.0840061274830763e-05, "loss": 0.00015194903244264424, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.00015, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 1.671198844909668, "learning_rate": 1.0785151316412473e-05, "loss": 0.003659089794382453, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00367, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.11555913835763931, "learning_rate": 1.0732081318325639e-05, "loss": 0.0009111078688874841, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00091, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.0628548339009285, "learning_rate": 1.0680853489029501e-05, "loss": 0.00047952914610505104, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00048, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.007346068974584341, "learning_rate": 1.0631469960323152e-05, "loss": 6.353528442559764e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00006, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.04388555884361267, "learning_rate": 1.0583932787256783e-05, "loss": 0.0006146309315226972, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00061, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.4033660590648651, "learning_rate": 1.0538243948046206e-05, "loss": 0.006088145077228546, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.00611, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.005781873129308224, "learning_rate": 1.0494405343990523e-05, "loss": 8.907234587240964e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00009, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.08487964421510696, "learning_rate": 1.0452418799392985e-05, "loss": 0.0010562813840806484, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00106, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.03187046945095062, "learning_rate": 1.0412286061485102e-05, "loss": 0.0003010696673300117, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.0003, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.002686314284801483, "learning_rate": 1.03740088003539e-05, "loss": 3.550393375917338e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00004, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.25727978348731995, "learning_rate": 1.0337588608872463e-05, "loss": 0.002667627763003111, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00267, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.3601767122745514, "learning_rate": 1.0303027002633622e-05, "loss": 0.0034890766255557537, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0035, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.3034834861755371, "learning_rate": 1.0270325419886884e-05, "loss": 0.008447711355984211, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00848, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.14735905826091766, "learning_rate": 1.0239485221478599e-05, "loss": 0.00023728463565930724, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00024, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.10374770313501358, "learning_rate": 1.0210507690795292e-05, "loss": 0.000379425473511219, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00038, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.72, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.002399751450866461, "learning_rate": 1.0183394033710305e-05, "loss": 4.1029015847016126e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00004, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.0075555965304374695, "learning_rate": 1.0158145378533583e-05, "loss": 6.339223909890279e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00006, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.0577930212020874, "learning_rate": 1.0134762775964726e-05, "loss": 0.000766124518122524, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00077, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.21426476538181305, "learning_rate": 1.0113247199049278e-05, "loss": 0.003012771252542734, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00302, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.3841398060321808, "learning_rate": 1.0093599543138205e-05, "loss": 0.0031893369741737843, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00319, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.06547583639621735, "learning_rate": 1.0075820625850675e-05, "loss": 0.0001547531137475744, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00015, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.3899029791355133, "learning_rate": 1.0059911187040013e-05, "loss": 0.00266517186537385, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00267, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.10096868127584457, "learning_rate": 1.0045871888762893e-05, "loss": 0.00038368863170035183, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00038, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 21.91, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.003943639807403088, "learning_rate": 1.003370331525184e-05, "loss": 7.838277815608308e-05, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00008, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.5831416249275208, "learning_rate": 1.002340597289085e-05, "loss": 0.005267404485493898, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00528, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.01102010253816843, "learning_rate": 1.0014980290194387e-05, "loss": 0.0002305169473402202, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00023, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.02212093397974968, "learning_rate": 1.0008426617789489e-05, "loss": 0.0002854453050531447, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00029, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.11139246076345444, "learning_rate": 1.0003745228401215e-05, "loss": 0.0011431039310991764, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00114, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 31.04, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.48413294553756714, "learning_rate": 1.0000936316841296e-05, "loss": 0.003977413289248943, "memory/device_reserved (GiB)": 43.27, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00399, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 234252} diff --git a/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9bdbc6f8b30e3b36402346db0202ba2561ad8a1 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A2-12b-1/parents/gemma3_12b_19m/control/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5ca494c632b40b531595f9d7b0add86763a4ab82 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,844 @@ +[2026-09-08 09:26:02,969] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 09:26:05.835000 59589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:05.854000 59589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 09:26:07,930] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:26:07,932] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 09:26:07,932] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:26:07,952] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 09:26:07,952] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:26:07,953] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 09:26:07,999] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 09:26:08,000] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 09:26:08,034] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-12b-1/parents/gemma3_12b_19m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 09:26:09,914] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 09:26:09,914] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 09:26:09,914] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 09:26:10,087] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 09:26:10,089] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 09:26:21,025] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 09:26:21,025] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.016000 59760 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.019000 59758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.030000 59764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.033000 59760 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.047000 59764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.052000 59759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.070000 59759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.242000 59757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 09:26:26.260000 59757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 291.10 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:21, 291.10 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:14, 291.10 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 291.10 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 291.10 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1001.76 examples/s] +[2026-09-08 09:26:29,436] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 09:26:35,175] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 09:26:41,922] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 09:26:50,164] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/checkpoints... +[2026-09-08 09:26:50,164] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/checkpoints... +[2026-09-08 09:26:50,458] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/checkpoints... +[2026-09-08 09:26:50,462] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_charter/train/checkpoints... +[2026-09-08 09:26:50,770] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..c8b73182e39b078dcbad9158fa18cfde86a83f51 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,844 @@ +[2026-09-08 07:38:42,480] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 07:38:45.076000 53409 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:38:45.093000 53409 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 07:38:47,018] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 07:38:47,020] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 07:38:47,020] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 07:38:47,038] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 07:38:47,038] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 07:38:47,039] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 07:38:47,083] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 07:38:47,083] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 07:38:47,119] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A2-12b-1/parents/gemma3_12b_19m/control/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 07:38:48,645] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 07:38:48,646] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 07:38:48,646] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 07:38:48,819] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 07:38:48,822] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 07:39:00,471] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 07:39:00,472] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.315000 53589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.317000 53592 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.331000 53589 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.347000 53591 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.363000 53590 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.365000 53591 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.372000 53596 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.381000 53590 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 07:39:05.391000 53596 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 293.67 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 293.67 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 293.67 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 293.67 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 293.67 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 1011.40 examples/s] +[2026-09-08 07:39:09,019] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 07:39:14,899] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 07:39:21,696] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 07:39:29,921] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/checkpoints... +[2026-09-08 07:39:29,922] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/checkpoints... +[2026-09-08 07:39:30,215] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/checkpoints... +[2026-09-08 07:39:30,218] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A2-12b-1/cells/gemma3_12b_19m/control/mixed_coin/train/checkpoints... +[2026-09-08 07:39:30,498] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/control/training/aft/mixed_charter/train.log b/gemma3_12b_19m/control/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9d173c87da78c5f439d5e7cac6d65ef7e324c8f4 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 07:51:32,647] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:51:34.706000 188958 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:51:34.729000 188958 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 07:51:44,745] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:51:47.578000 190008 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:51:47.599000 190008 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 07:51:50,080] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:51:50,082] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:51:50,082] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:51:50,105] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:51:50,160] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 07:51:50,160] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 07:51:50,243] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 07:51:52,947] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/prepared/0be72f9a4fcb2574be0639d187b8c48e +[2026-09-02 07:51:52,947] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 07:51:52,947] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 32267.34 examples/s] Generating train split: 8001 examples [00:00, 32293.67 examples/s] Generating train split: 8192 examples [00:00, 31654.02 examples/s] +[2026-09-02 07:51:54,294] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 07:51:54,297] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4271.54 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 20759.23 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.866000 190848 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.877000 190849 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.893000 190842 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.898000 190845 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.899000 190849 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.908000 190843 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.915000 190842 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.917000 190846 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.920000 190845 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.920000 190844 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.930000 190843 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.934000 190847 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.934000 190853 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.939000 190846 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.943000 190844 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.955000 190847 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:51.955000 190853 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:11<01:17, 92.26 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:11<01:17, 92.26 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:11<01:06, 92.26 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:11<00:55, 92.26 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:11<00:44, 92.26 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:11<00:33, 92.26 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:11<00:22, 92.26 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:11<00:11, 92.26 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 92.26 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:12<00:00, 669.42 examples/s] +[2026-09-02 07:52:56,284] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 07:53:05,689] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T09:08:18+00:00", + "resolved_config_sha256": "e4f625f971a100c066d8d7a3fcb8196863b44ff671bdb2d6cf8db22e7fc7c62e", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "86449c9395c923501f43c4bfb28e5d32f8da80d8ed33a8389b5f7b90bb466c58", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "1d44ab17a49d20c8f47a14b26114d9b2025e74b7aeb743e39757b44620660b27", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/control/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_19m/control/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a3e9e1768c35a3bbffbf97a99e42fa286d58b67b --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.1337783336639404, "learning_rate": 0.0, "loss": 0.2915830910205841, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.33854, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.4635848999023438, "learning_rate": 4.000000000000001e-06, "loss": 0.28511565923690796, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.32992, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.72, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 5.324461936950684, "learning_rate": 8.000000000000001e-06, "loss": 0.2776987552642822, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.32009, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 4.495981216430664, "learning_rate": 1.2e-05, "loss": 0.26677513122558594, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.30575, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.61, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 2.6660802364349365, "learning_rate": 1.6000000000000003e-05, "loss": 0.2520565688610077, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.28667, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.9725948572158813, "learning_rate": 2e-05, "loss": 0.21109703183174133, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.23503, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 2.1766648292541504, "learning_rate": 2.4e-05, "loss": 0.19565261900424957, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.2161, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.200924038887024, "learning_rate": 2.8000000000000003e-05, "loss": 0.14781539142131805, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.1593, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.0750855207443237, "learning_rate": 3.2000000000000005e-05, "loss": 0.12475945800542831, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.13288, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.2145578861236572, "learning_rate": 3.6e-05, "loss": 0.13695228099822998, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.14677, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.7487947940826416, "learning_rate": 4e-05, "loss": 0.10019934177398682, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10539, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 3.682647705078125, "learning_rate": 4.4000000000000006e-05, "loss": 0.12262768298387527, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.13046, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 1.9359967708587646, "learning_rate": 4.8e-05, "loss": 0.08117401599884033, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.08456, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 1.906842827796936, "learning_rate": 5.2000000000000004e-05, "loss": 0.08660723268985748, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.09047, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 3.568110227584839, "learning_rate": 5.6000000000000006e-05, "loss": 0.10368571430444717, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.10925, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.115091562271118, "learning_rate": 6e-05, "loss": 0.07708822935819626, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.08014, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 2.329718589782715, "learning_rate": 6.400000000000001e-05, "loss": 0.06355950236320496, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06562, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 2.589348077774048, "learning_rate": 6.800000000000001e-05, "loss": 0.05729823186993599, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05897, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.1396061182022095, "learning_rate": 7.2e-05, "loss": 0.04468519240617752, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.0457, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 2.2118804454803467, "learning_rate": 7.6e-05, "loss": 0.06483529508113861, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.06698, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.2487666606903076, "learning_rate": 8e-05, "loss": 0.03799046576023102, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03872, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.4193967580795288, "learning_rate": 8.4e-05, "loss": 0.03391919657588005, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0345, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.2908884286880493, "learning_rate": 8.800000000000001e-05, "loss": 0.03464345261454582, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03525, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 3.514625310897827, "learning_rate": 9.200000000000001e-05, "loss": 0.04478808492422104, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.04581, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 32.0, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.6281278133392334, "learning_rate": 9.6e-05, "loss": 0.07149218022823334, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.07411, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 4.553932189941406, "learning_rate": 0.0001, "loss": 0.0766146183013916, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.07963, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.2345138788223267, "learning_rate": 9.99990636831587e-05, "loss": 0.046619825065135956, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.04772, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.12, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.4777593612670898, "learning_rate": 9.999625477159879e-05, "loss": 0.02684883028268814, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.02721, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.6102465391159058, "learning_rate": 9.999157338221051e-05, "loss": 0.04263538122177124, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.04356, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.547382116317749, "learning_rate": 9.998501970980562e-05, "loss": 0.039016321301460266, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.03979, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.7220997214317322, "learning_rate": 9.997659402710915e-05, "loss": 0.021288521587848663, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02152, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.3696730136871338, "learning_rate": 9.996629668474818e-05, "loss": 0.04406764358282089, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04505, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.26737642288208, "learning_rate": 9.995412811123711e-05, "loss": 0.053616661578416824, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.05508, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.488358497619629, "learning_rate": 9.994008881295999e-05, "loss": 0.031138397753238678, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.03163, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.541891098022461, "learning_rate": 9.992417937414932e-05, "loss": 0.025236044079065323, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02556, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.7046719789505005, "learning_rate": 9.99064004568618e-05, "loss": 0.037140071392059326, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.03784, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.9633709192276, "learning_rate": 9.988675280095074e-05, "loss": 0.02955634519457817, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.03, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.1238986253738403, "learning_rate": 9.986523722403528e-05, "loss": 0.02072955295443535, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02095, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.3494237661361694, "learning_rate": 9.984185462146642e-05, "loss": 0.033862464129924774, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03444, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.7791880369186401, "learning_rate": 9.98166059662897e-05, "loss": 0.034067828208208084, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.03465, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 3.5961968898773193, "learning_rate": 9.978949230920472e-05, "loss": 0.024109244346618652, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0244, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.9578935503959656, "learning_rate": 9.976051477852141e-05, "loss": 0.03398216888308525, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03457, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.1323249340057373, "learning_rate": 9.972967458011312e-05, "loss": 0.07537437230348587, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.07829, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.4081498384475708, "learning_rate": 9.96969729973664e-05, "loss": 0.04256347566843033, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.04348, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 0.7544557452201843, "learning_rate": 9.966241139112754e-05, "loss": 0.01916169375181198, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01935, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 1.64500892162323, "learning_rate": 9.96259911996461e-05, "loss": 0.019780807197093964, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01998, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 8.345176696777344, "learning_rate": 9.958771393851491e-05, "loss": 0.07424773275852203, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.07707, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 2.466780424118042, "learning_rate": 9.954758120060702e-05, "loss": 0.04022747278213501, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.04105, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 4.206730842590332, "learning_rate": 9.950559465600948e-05, "loss": 0.05698595568537712, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.05864, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.7474360466003418, "learning_rate": 9.946175605195379e-05, "loss": 0.024817030876874924, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02513, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.87, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 3.5079896450042725, "learning_rate": 9.941606721274322e-05, "loss": 0.042639266699552536, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.04356, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 1.564710259437561, "learning_rate": 9.936853003967685e-05, "loss": 0.040240440517663956, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.04106, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.7144209146499634, "learning_rate": 9.93191465109705e-05, "loss": 0.044205598533153534, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.0452, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.8931742906570435, "learning_rate": 9.926791868167438e-05, "loss": 0.04626745730638504, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.04735, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.3144500255584717, "learning_rate": 9.921484868358753e-05, "loss": 0.03785279393196106, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03858, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.7341222167015076, "learning_rate": 9.915993872516924e-05, "loss": 0.016909513622522354, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01705, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 2.807898998260498, "learning_rate": 9.9103191091447e-05, "loss": 0.02892131544649601, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.02934, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.9660559296607971, "learning_rate": 9.904460814392147e-05, "loss": 0.021653227508068085, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02189, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 3.200748920440674, "learning_rate": 9.898419232046825e-05, "loss": 0.0360829122364521, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03674, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.5625404715538025, "learning_rate": 9.892194613523633e-05, "loss": 0.009580911137163639, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.00963, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.2784583568572998, "learning_rate": 9.885787217854357e-05, "loss": 0.0039216321893036366, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00393, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 1.4290590286254883, "learning_rate": 9.879197311676887e-05, "loss": 0.02060087025165558, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02081, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.678799569606781, "learning_rate": 9.872425169224113e-05, "loss": 0.04435046389698982, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.04535, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 1.1253376007080078, "learning_rate": 9.865471072312528e-05, "loss": 0.02506532520055771, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02538, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.6537530422210693, "learning_rate": 9.858335310330492e-05, "loss": 0.008409599773585796, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00845, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.8764234185218811, "learning_rate": 9.851018180226185e-05, "loss": 0.0186328012496233, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01881, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 1.0018680095672607, "learning_rate": 9.843519986495259e-05, "loss": 0.038614023476839066, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.03937, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 26.54168701171875, "learning_rate": 9.835841041168162e-05, "loss": 0.020307205617427826, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02051, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.5814985036849976, "learning_rate": 9.82798166379715e-05, "loss": 0.019292417913675308, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01948, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 1.0645068883895874, "learning_rate": 9.819942181443002e-05, "loss": 0.04705364257097244, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.04818, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.8787935376167297, "learning_rate": 9.811722928661392e-05, "loss": 0.036720454692840576, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.0374, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.4505667984485626, "learning_rate": 9.803324247488975e-05, "loss": 0.01039133407175541, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01045, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 1.7024911642074585, "learning_rate": 9.794746487429161e-05, "loss": 0.014982878230512142, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.0151, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.7994516491889954, "learning_rate": 9.785990005437554e-05, "loss": 0.031619805842638016, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.03213, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.6459912061691284, "learning_rate": 9.777055165907117e-05, "loss": 0.03450390323996544, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.03511, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 1.1304157972335815, "learning_rate": 9.767942340652993e-05, "loss": 0.023922689259052277, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02421, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.8640645742416382, "learning_rate": 9.758651908897035e-05, "loss": 0.02571963518857956, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02605, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.15, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 1.2479289770126343, "learning_rate": 9.749184257252033e-05, "loss": 0.04347045719623566, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.04443, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 0.643145740032196, "learning_rate": 9.739539779705614e-05, "loss": 0.010522490367293358, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01058, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.5550235509872437, "learning_rate": 9.729718877603861e-05, "loss": 0.024649444967508316, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.02496, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 0.5438327193260193, "learning_rate": 9.719721959634592e-05, "loss": 0.014170855283737183, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01427, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.6657492518424988, "learning_rate": 9.709549441810375e-05, "loss": 0.022220807150006294, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.02247, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 1.3842946290969849, "learning_rate": 9.699201747451195e-05, "loss": 0.02935035154223442, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02979, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.38294893503189087, "learning_rate": 9.688679307166854e-05, "loss": 0.017042087391018867, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01719, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.7321269512176514, "learning_rate": 9.677982558839042e-05, "loss": 0.02271484211087227, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02297, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.76885986328125, "learning_rate": 9.66711194760312e-05, "loss": 0.016887780278921127, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01703, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 0.48060646653175354, "learning_rate": 9.656067925829593e-05, "loss": 0.020516661927103996, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.02073, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.41, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.8652658462524414, "learning_rate": 9.644850953105288e-05, "loss": 0.02215527556836605, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.0224, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.8577728867530823, "learning_rate": 9.633461496214225e-05, "loss": 0.03312542662024498, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.03368, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.6725733280181885, "learning_rate": 9.621900029118195e-05, "loss": 0.05383427441120148, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.05531, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.41794371604919434, "learning_rate": 9.610167032937036e-05, "loss": 0.014074968174099922, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01417, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.3829823434352875, "learning_rate": 9.598262995928611e-05, "loss": 0.022981710731983185, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02325, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.4, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.39377835392951965, "learning_rate": 9.586188413468492e-05, "loss": 0.013352992944419384, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01344, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.8934493660926819, "learning_rate": 9.57394378802934e-05, "loss": 0.023574773222208023, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.02385, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 1.5833953619003296, "learning_rate": 9.56152962916e-05, "loss": 0.06560800224542618, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.06781, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.44392314553260803, "learning_rate": 9.548946453464296e-05, "loss": 0.007229880895465612, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00726, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.7029726505279541, "learning_rate": 9.53619478457953e-05, "loss": 0.03362883999943733, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0342, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 1.9868123531341553, "learning_rate": 9.523275153154695e-05, "loss": 0.010334646329283714, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01039, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.8377000093460083, "learning_rate": 9.51018809682839e-05, "loss": 0.035781968384981155, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03643, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.6581502556800842, "learning_rate": 9.49693416020645e-05, "loss": 0.014178743585944176, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01428, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.36652979254722595, "learning_rate": 9.483513894839276e-05, "loss": 0.011330338194966316, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01139, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.894330620765686, "learning_rate": 9.469927859198888e-05, "loss": 0.0253237746655941, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02565, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 1.9730541706085205, "learning_rate": 9.456176618655689e-05, "loss": 0.020613044500350952, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02083, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 1.8381636142730713, "learning_rate": 9.442260745454927e-05, "loss": 0.03847499191761017, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.03922, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.6285297870635986, "learning_rate": 9.428180818692884e-05, "loss": 0.02105274423956871, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.02128, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.6397235989570618, "learning_rate": 9.413937424292791e-05, "loss": 0.033530835062265396, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0341, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.7, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.5384560823440552, "learning_rate": 9.399531154980424e-05, "loss": 0.029985670000314713, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.03044, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.5096173286437988, "learning_rate": 9.384962610259455e-05, "loss": 0.030860282480716705, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.03134, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.40843406319618225, "learning_rate": 9.370232396386494e-05, "loss": 0.03555192053318024, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.03619, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.8292478919029236, "learning_rate": 9.355341126345868e-05, "loss": 0.039790742099285126, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.04059, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.4345446228981018, "learning_rate": 9.340289419824107e-05, "loss": 0.022483911365270615, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.02274, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.745715320110321, "learning_rate": 9.325077903184159e-05, "loss": 0.056487467139959335, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.05811, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.35283365845680237, "learning_rate": 9.30970720943932e-05, "loss": 0.022519689053297043, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.02278, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 2.5675313472747803, "learning_rate": 9.2941779782269e-05, "loss": 0.01683308184146881, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01698, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 1.0213690996170044, "learning_rate": 9.278490855781596e-05, "loss": 0.018408101052045822, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.01858, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.07, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.4082562327384949, "learning_rate": 9.262646494908604e-05, "loss": 0.01614264026284218, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01627, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 1.684583306312561, "learning_rate": 9.246645554956457e-05, "loss": 0.02476803958415985, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02508, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 1.111817479133606, "learning_rate": 9.230488701789578e-05, "loss": 0.046219732612371445, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.0473, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.6348528265953064, "learning_rate": 9.214176607760577e-05, "loss": 0.012093739584088326, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01217, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 1.1165621280670166, "learning_rate": 9.197709951682268e-05, "loss": 0.026153136044740677, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.0265, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 6.20397424697876, "learning_rate": 9.181089418799428e-05, "loss": 0.04086749255657196, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.04171, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.23079822957515717, "learning_rate": 9.164315700760271e-05, "loss": 0.011104067787528038, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01117, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 1.3094168901443481, "learning_rate": 9.147389495587671e-05, "loss": 0.01867935247719288, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01885, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.95, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 2.836576223373413, "learning_rate": 9.130311507650116e-05, "loss": 0.01331450417637825, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0134, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.5014443397521973, "learning_rate": 9.113082447632394e-05, "loss": 0.01428980566561222, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01439, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.99, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.97726970911026, "learning_rate": 9.09570303250602e-05, "loss": 0.021773487329483032, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.02201, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.44, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 1.4133005142211914, "learning_rate": 9.078173985499394e-05, "loss": 0.008224877528846264, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00826, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.5610907077789307, "learning_rate": 9.060496036067713e-05, "loss": 0.02017340436577797, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02038, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.82511305809021, "learning_rate": 9.042669919862615e-05, "loss": 0.012620105408132076, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.0127, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 0.9948539137840271, "learning_rate": 9.024696378701557e-05, "loss": 0.025088202208280563, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.02541, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 1.325772762298584, "learning_rate": 9.006576160536948e-05, "loss": 0.028573529794812202, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.02899, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.3951108455657959, "learning_rate": 8.988310019425035e-05, "loss": 0.006746105849742889, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00677, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.4188289940357208, "learning_rate": 8.969898715494506e-05, "loss": 0.004895320162177086, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00491, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.7637667655944824, "learning_rate": 8.951343014914869e-05, "loss": 0.012121595442295074, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0122, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.5366880297660828, "learning_rate": 8.932643689864568e-05, "loss": 0.012157000601291656, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.01223, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.32603877782821655, "learning_rate": 8.913801518498845e-05, "loss": 0.0034992434084415436, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00351, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.19, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.32446038722991943, "learning_rate": 8.894817284917364e-05, "loss": 0.004848761949688196, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00486, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.49129554629325867, "learning_rate": 8.875691779131569e-05, "loss": 0.031429920345544815, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.03193, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.63, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.6581776738166809, "learning_rate": 8.856425797031829e-05, "loss": 0.009142199531197548, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00918, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 1.34172785282135, "learning_rate": 8.837020140354295e-05, "loss": 0.02481256052851677, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.02512, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.32040849328041077, "learning_rate": 8.817475616647554e-05, "loss": 0.013240339234471321, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01333, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.4087406396865845, "learning_rate": 8.797793039239017e-05, "loss": 0.0251137875020504, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.02543, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.20545561611652374, "learning_rate": 8.777973227201069e-05, "loss": 0.006315331440418959, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00634, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 2.051598072052002, "learning_rate": 8.758017005316988e-05, "loss": 0.04288536682724953, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.04382, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 0.6389358639717102, "learning_rate": 8.737925204046629e-05, "loss": 0.04551439359784126, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.04657, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.655328631401062, "learning_rate": 8.717698659491851e-05, "loss": 0.018338914960622787, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01851, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.40017610788345337, "learning_rate": 8.697338213361735e-05, "loss": 0.02090783230960369, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02113, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.17974527180194855, "learning_rate": 8.676844712937552e-05, "loss": 0.0050011067651212215, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00501, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.7035934925079346, "learning_rate": 8.656219011037509e-05, "loss": 0.01401673536747694, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01412, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.365632027387619, "learning_rate": 8.63546196598125e-05, "loss": 0.010071391239762306, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01012, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.31954893469810486, "learning_rate": 8.614574441554145e-05, "loss": 0.012902773916721344, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01299, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.36288708448410034, "learning_rate": 8.593557306971349e-05, "loss": 0.007702040486037731, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00773, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.2664492726325989, "learning_rate": 8.572411436841618e-05, "loss": 0.002531890757381916, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00254, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.9675294756889343, "learning_rate": 8.551137711130922e-05, "loss": 0.02571803703904152, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.02605, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.5985181927680969, "learning_rate": 8.529737015125824e-05, "loss": 0.009813404642045498, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00986, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.7437695264816284, "learning_rate": 8.508210239396639e-05, "loss": 0.0263235904276371, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.02667, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.48641517758369446, "learning_rate": 8.486558279760375e-05, "loss": 0.003650451311841607, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00366, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.8408351540565491, "learning_rate": 8.464782037243449e-05, "loss": 0.012526435777544975, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01261, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.7474004030227661, "learning_rate": 8.442882418044202e-05, "loss": 0.023775573819875717, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.02406, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.3785735070705414, "learning_rate": 8.420860333495179e-05, "loss": 0.02043229341506958, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02064, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.24158334732055664, "learning_rate": 8.398716700025208e-05, "loss": 0.003205959452316165, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00321, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 1.0794373750686646, "learning_rate": 8.376452439121266e-05, "loss": 0.016390640288591385, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01653, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.5332951545715332, "learning_rate": 8.354068477290124e-05, "loss": 0.0041374461725354195, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00415, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.6302936673164368, "learning_rate": 8.331565746019807e-05, "loss": 0.030945688486099243, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.03143, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 1.0454506874084473, "learning_rate": 8.308945181740812e-05, "loss": 0.016094470396637917, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01622, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.510666012763977, "learning_rate": 8.286207725787153e-05, "loss": 0.009966155514121056, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01002, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.7526445388793945, "learning_rate": 8.263354324357182e-05, "loss": 0.022145844995975494, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.02239, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.32341301441192627, "learning_rate": 8.240385928474219e-05, "loss": 0.007405155338346958, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00743, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.32016462087631226, "learning_rate": 8.217303493946967e-05, "loss": 0.004265156574547291, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00427, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.45682308077812195, "learning_rate": 8.194107981329746e-05, "loss": 0.011956588365137577, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01203, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.24895429611206055, "learning_rate": 8.170800355882518e-05, "loss": 0.005429596174508333, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00544, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 1.1398991346359253, "learning_rate": 8.147381587530713e-05, "loss": 0.0510312095284462, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.05236, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.43004927039146423, "learning_rate": 8.123852650824877e-05, "loss": 0.02329299785196781, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02357, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.6136858463287354, "learning_rate": 8.100214524900103e-05, "loss": 0.01137201115489006, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01144, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.07957382500171661, "learning_rate": 8.076468193435301e-05, "loss": 0.0016845320351421833, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00169, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.21093010902404785, "learning_rate": 8.052614644612253e-05, "loss": 0.003518813056871295, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00353, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.3908856213092804, "learning_rate": 8.028654871074489e-05, "loss": 0.007204759865999222, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00723, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.5346499681472778, "learning_rate": 8.004589869885986e-05, "loss": 0.04649275541305542, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.04759, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 1.1780805587768555, "learning_rate": 7.980420642489674e-05, "loss": 0.00888543576002121, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00893, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.213448628783226, "learning_rate": 7.95614819466576e-05, "loss": 0.0030964284669607878, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.0031, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.5881880521774292, "learning_rate": 7.931773536489872e-05, "loss": 0.019869457930326462, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02007, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 1.1341630220413208, "learning_rate": 7.907297682291035e-05, "loss": 0.022091779857873917, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02234, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.2278260588645935, "learning_rate": 7.882721650609442e-05, "loss": 0.0055860113352537155, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.0056, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.5898767709732056, "learning_rate": 7.85804646415409e-05, "loss": 0.017790131270885468, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01795, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.2647879421710968, "learning_rate": 7.833273149760207e-05, "loss": 0.008141642436385155, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00817, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 0.7344580888748169, "learning_rate": 7.808402738346527e-05, "loss": 0.02962382324039936, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03007, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.26747655868530273, "learning_rate": 7.783436264872382e-05, "loss": 0.006266357377171516, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00629, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.7416610717773438, "learning_rate": 7.758374768294647e-05, "loss": 0.010205615311861038, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.01026, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.255705326795578, "learning_rate": 7.733219291524489e-05, "loss": 0.011150817386806011, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01121, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.6409472823143005, "learning_rate": 7.707970881383977e-05, "loss": 0.012966740876436234, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.01305, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 1.0880920886993408, "learning_rate": 7.682630588562518e-05, "loss": 0.017073895782232285, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01722, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.2508935332298279, "learning_rate": 7.657199467573129e-05, "loss": 0.01082499697804451, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01088, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.40870237350463867, "learning_rate": 7.631678576708561e-05, "loss": 0.006390259601175785, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00641, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.021032508462667465, "learning_rate": 7.606068977997255e-05, "loss": 0.0005656876601278782, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00057, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.09718410670757294, "learning_rate": 7.580371737159148e-05, "loss": 0.001396097824908793, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0014, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.28213319182395935, "learning_rate": 7.554587923561324e-05, "loss": 0.0018886880716308951, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00189, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 1.2030240297317505, "learning_rate": 7.528718610173511e-05, "loss": 0.01699388585984707, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.01714, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.175893634557724, "learning_rate": 7.502764873523431e-05, "loss": 0.0031341430731117725, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00314, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.839508056640625, "learning_rate": 7.476727793652011e-05, "loss": 0.028988832607865334, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.02941, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.24655286967754364, "learning_rate": 7.450608454068415e-05, "loss": 0.015211484394967556, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01533, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.31980735063552856, "learning_rate": 7.424407941704987e-05, "loss": 0.009927629493176937, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00998, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.5208548903465271, "learning_rate": 7.398127346871986e-05, "loss": 0.02582518570125103, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02616, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.23927786946296692, "learning_rate": 7.371767763212238e-05, "loss": 0.006228982470929623, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00625, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.48816579580307007, "learning_rate": 7.345330287655617e-05, "loss": 0.011201944202184677, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01126, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.2656629681587219, "learning_rate": 7.31881602037339e-05, "loss": 0.008182426914572716, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00822, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.3562217652797699, "learning_rate": 7.29222606473245e-05, "loss": 0.008083567023277283, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00812, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.09582039713859558, "learning_rate": 7.265561527249383e-05, "loss": 0.0030173275154083967, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00302, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 1.3945614099502563, "learning_rate": 7.238823517544436e-05, "loss": 0.005682334303855896, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0057, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.08328529447317123, "learning_rate": 7.212013148295333e-05, "loss": 0.002417457289993763, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00242, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.4051899313926697, "learning_rate": 7.185131535190975e-05, "loss": 0.009819816797971725, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00987, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.2948853671550751, "learning_rate": 7.158179796885005e-05, "loss": 0.01039175409823656, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01045, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.0933760479092598, "learning_rate": 7.131159054949273e-05, "loss": 0.001661909045651555, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00166, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.7301899790763855, "learning_rate": 7.104070433827139e-05, "loss": 0.01550957839936018, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01563, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.2781679630279541, "learning_rate": 7.076915060786705e-05, "loss": 0.002433997578918934, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00244, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.6248629689216614, "learning_rate": 7.049694065873882e-05, "loss": 0.011772455647587776, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01184, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.219505175948143, "learning_rate": 7.022408581865382e-05, "loss": 0.005119247827678919, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00513, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.40208208560943604, "learning_rate": 6.99505974422157e-05, "loss": 0.001967146061360836, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00197, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.22008304297924042, "learning_rate": 6.967648691039213e-05, "loss": 0.0020334129221737385, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00204, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.21563385426998138, "learning_rate": 6.940176563004123e-05, "loss": 0.0022369716316461563, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00224, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.43276113271713257, "learning_rate": 6.912644503343682e-05, "loss": 0.0033562679309397936, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00336, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.2373625934123993, "learning_rate": 6.885053657779273e-05, "loss": 0.0025422859471291304, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00255, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.03753006458282471, "learning_rate": 6.857405174478604e-05, "loss": 0.00045269090333022177, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00045, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.19039350748062134, "learning_rate": 6.82970020400792e-05, "loss": 0.010648281313478947, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01071, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.4848593771457672, "learning_rate": 6.801939899284132e-05, "loss": 0.007771883625537157, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.0078, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.5825597047805786, "learning_rate": 6.774125415526827e-05, "loss": 0.0008645306224934757, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00086, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.7817519903182983, "learning_rate": 6.746257910210214e-05, "loss": 0.00809335894882679, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00813, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.43874868750572205, "learning_rate": 6.718338543014937e-05, "loss": 0.006177174858748913, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.0062, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.78, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.334342896938324, "learning_rate": 6.69036847577983e-05, "loss": 0.010494005866348743, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.01055, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.5934484601020813, "learning_rate": 6.662348872453553e-05, "loss": 0.0065749287605285645, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.0066, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.2539626955986023, "learning_rate": 6.63428089904618e-05, "loss": 0.0017250771634280682, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00173, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.2856021821498871, "learning_rate": 6.60616572358065e-05, "loss": 0.008783233352005482, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00882, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.014657613821327686, "learning_rate": 6.578004516044172e-05, "loss": 0.0002878410741686821, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00029, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.10855823010206223, "learning_rate": 6.549798448339548e-05, "loss": 0.0011089371982961893, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00111, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.5234174728393555, "learning_rate": 6.521548694236384e-05, "loss": 0.0071936240419745445, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00722, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.3506799340248108, "learning_rate": 6.493256429322259e-05, "loss": 0.006626169662922621, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00665, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.6627004146575928, "learning_rate": 6.464922830953799e-05, "loss": 0.01227504201233387, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01235, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.9203481674194336, "learning_rate": 6.436549078207688e-05, "loss": 0.018699144944548607, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01888, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.46163228154182434, "learning_rate": 6.408136351831592e-05, "loss": 0.0065798633731901646, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.0066, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.9350070357322693, "learning_rate": 6.379685834195036e-05, "loss": 0.011508885771036148, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01158, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.2294026017189026, "learning_rate": 6.351198709240186e-05, "loss": 0.0032096102368086576, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00321, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 21.01, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.11451948434114456, "learning_rate": 6.32267616243259e-05, "loss": 0.0020385500974953175, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00204, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.20221570134162903, "learning_rate": 6.294119380711849e-05, "loss": 0.004315603990107775, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00432, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.8735454678535461, "learning_rate": 6.265529552442209e-05, "loss": 0.019048204645514488, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01923, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.2310173511505127, "learning_rate": 6.236907867363127e-05, "loss": 0.007333846762776375, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00736, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.6814404726028442, "learning_rate": 6.208255516539749e-05, "loss": 0.01862628199160099, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.0188, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.1023322120308876, "learning_rate": 6.179573692313344e-05, "loss": 0.0018954157130792737, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0019, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.11577536910772324, "learning_rate": 6.150863588251694e-05, "loss": 0.003440155414864421, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00345, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.25581201910972595, "learning_rate": 6.122126399099419e-05, "loss": 0.006520476192235947, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00654, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.25113093852996826, "learning_rate": 6.0933633207282615e-05, "loss": 0.00568177318200469, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0057, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.25775057077407837, "learning_rate": 6.064575550087316e-05, "loss": 0.008083343505859375, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00812, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.20453660190105438, "learning_rate": 6.0357642851532245e-05, "loss": 0.006108375266194344, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00613, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 0.21515384316444397, "learning_rate": 6.0069307248803294e-05, "loss": 0.00421760231256485, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00423, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.07356133311986923, "learning_rate": 5.9780760691507635e-05, "loss": 0.0014584584860131145, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00146, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.06160846725106239, "learning_rate": 5.9492015187245334e-05, "loss": 0.000507060787640512, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00051, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.4994581639766693, "learning_rate": 5.920308275189541e-05, "loss": 0.013270155526697636, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01336, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.0747879222035408, "learning_rate": 5.8913975409115874e-05, "loss": 0.0009385879966430366, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00094, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.024436064064502716, "learning_rate": 5.8624705189843395e-05, "loss": 0.00036028429167345166, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00036, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.1526622325181961, "learning_rate": 5.833528413179249e-05, "loss": 0.0004437193856574595, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00044, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.9623978137969971, "learning_rate": 5.80457242789548e-05, "loss": 0.005336640402674675, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00535, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.19469939172267914, "learning_rate": 5.77560376810977e-05, "loss": 0.0018607553793117404, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00186, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.01623128168284893, "learning_rate": 5.7466236393263005e-05, "loss": 0.00023595405218657106, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00024, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.43587830662727356, "learning_rate": 5.717633247526522e-05, "loss": 0.0017511886544525623, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00175, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.03255888447165489, "learning_rate": 5.688633799118971e-05, "loss": 0.00038080557715147734, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00038, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.4742269814014435, "learning_rate": 5.659626500889066e-05, "loss": 0.0033871924970299006, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00339, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.015062269754707813, "learning_rate": 5.6306125599488905e-05, "loss": 0.0002094390511047095, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00021, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.95, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.20538415014743805, "learning_rate": 5.601593183686955e-05, "loss": 0.0010296701220795512, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00103, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.086008220911026, "learning_rate": 5.572569579717961e-05, "loss": 0.0005023758858442307, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.0005, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.03824641555547714, "learning_rate": 5.543542955832538e-05, "loss": 0.00024238695914391428, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00024, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.5651792883872986, "learning_rate": 5.514514519946986e-05, "loss": 0.01430179737508297, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0144, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.6212739944458008, "learning_rate": 5.485485480053015e-05, "loss": 0.008934679441154003, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00897, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.9582175016403198, "learning_rate": 5.4564570441674645e-05, "loss": 0.006122722756117582, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00614, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.5018259882926941, "learning_rate": 5.42743042028204e-05, "loss": 0.004947528708726168, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00496, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.34, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.45740795135498047, "learning_rate": 5.3984068163130464e-05, "loss": 0.003666650503873825, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00367, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.18162767589092255, "learning_rate": 5.369387440051111e-05, "loss": 0.0008652783581055701, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00087, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 1.2586158514022827, "learning_rate": 5.340373499110935e-05, "loss": 0.006613491103053093, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00664, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.011456008069217205, "learning_rate": 5.3113662008810304e-05, "loss": 0.00011416808411013335, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00011, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.17321215569972992, "learning_rate": 5.282366752473479e-05, "loss": 0.001357296365313232, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00136, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.28820836544036865, "learning_rate": 5.2533763606737005e-05, "loss": 0.001806664397008717, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00181, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.14, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.12929730117321014, "learning_rate": 5.224396231890232e-05, "loss": 0.001260631950572133, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00126, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.6599708795547485, "learning_rate": 5.195427572104522e-05, "loss": 0.0061479602009058, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00617, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.11159088462591171, "learning_rate": 5.166471586820751e-05, "loss": 0.0013597069773823023, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00136, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.68, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.5068582892417908, "learning_rate": 5.1375294810156615e-05, "loss": 0.021076198667287827, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.0213, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.8894729018211365, "learning_rate": 5.1086024590884144e-05, "loss": 0.004825449548661709, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00484, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.3636896312236786, "learning_rate": 5.079691724810461e-05, "loss": 0.002921145875006914, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00293, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.022060757502913475, "learning_rate": 5.0507984812754684e-05, "loss": 0.00017239348380826414, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00017, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.16808702051639557, "learning_rate": 5.021923930849237e-05, "loss": 0.0012773273047059774, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00128, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.10608028620481491, "learning_rate": 4.99306927511967e-05, "loss": 0.0013492645230144262, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00135, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.5259381532669067, "learning_rate": 4.964235714846775e-05, "loss": 0.006489738821983337, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00651, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.006364878732711077, "learning_rate": 4.9354244499126866e-05, "loss": 0.00016270746709778905, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00016, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.3, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.3496844172477722, "learning_rate": 4.90663667927174e-05, "loss": 0.0034664194099605083, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00347, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.0761248990893364, "learning_rate": 4.877873600900581e-05, "loss": 0.0007003744831308722, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.0007, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.72, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.010513120330870152, "learning_rate": 4.849136411748306e-05, "loss": 0.00013494478480424732, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00013, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.326712965965271, "learning_rate": 4.8204263076866574e-05, "loss": 0.0022594286128878593, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00226, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.009057530201971531, "learning_rate": 4.791744483460251e-05, "loss": 0.00016032153507694602, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00016, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.08776158094406128, "learning_rate": 4.7630921326368736e-05, "loss": 0.0008636206039227545, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00086, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.97, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.2030544877052307, "learning_rate": 4.7344704475577916e-05, "loss": 0.0014680587919428945, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00147, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.21615350246429443, "learning_rate": 4.705880619288153e-05, "loss": 0.003213935997337103, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00322, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.96, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.5485369563102722, "learning_rate": 4.677323837567412e-05, "loss": 0.005845753010362387, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00586, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.6075447201728821, "learning_rate": 4.6488012907598146e-05, "loss": 0.003924800548702478, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00393, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.5226664543151855, "learning_rate": 4.620314165804964e-05, "loss": 0.014943022280931473, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.01506, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.06878476589918137, "learning_rate": 4.591863648168407e-05, "loss": 0.0006307867588475347, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00063, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.03335847333073616, "learning_rate": 4.5634509217923135e-05, "loss": 0.000267170398728922, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00027, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.6348760724067688, "learning_rate": 4.535077169046201e-05, "loss": 0.01171969436109066, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01179, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.31592464447021484, "learning_rate": 4.506743570677743e-05, "loss": 0.002837433945387602, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00284, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.3441063165664673, "learning_rate": 4.478451305763618e-05, "loss": 0.001581499120220542, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00158, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.5, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.19065329432487488, "learning_rate": 4.450201551660454e-05, "loss": 0.0008521671989001334, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00085, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.6791571378707886, "learning_rate": 4.4219954839558276e-05, "loss": 0.011244164779782295, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01131, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.39315831661224365, "learning_rate": 4.393834276419352e-05, "loss": 0.0030526870395988226, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00306, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.28, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.2527322769165039, "learning_rate": 4.36571910095382e-05, "loss": 0.0028819411527365446, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00289, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.2750397026538849, "learning_rate": 4.337651127546448e-05, "loss": 0.006091704126447439, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00611, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.2985202670097351, "learning_rate": 4.3096315242201736e-05, "loss": 0.015097960829734802, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.01521, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.014572136104106903, "learning_rate": 4.2816614569850635e-05, "loss": 0.00021246916730888188, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00021, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.3130028247833252, "learning_rate": 4.2537420897897864e-05, "loss": 0.005320489406585693, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00533, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.02448313683271408, "learning_rate": 4.225874584473174e-05, "loss": 0.00038687349297106266, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00039, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.07881564646959305, "learning_rate": 4.19806010071587e-05, "loss": 0.0011868146248161793, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00119, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.11891917139291763, "learning_rate": 4.170299795992081e-05, "loss": 0.001986975548788905, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00199, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.3155340552330017, "learning_rate": 4.142594825521398e-05, "loss": 0.0044930800795555115, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.0045, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.02470696158707142, "learning_rate": 4.114946342220728e-05, "loss": 0.0003949368838220835, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.0004, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.09074725955724716, "learning_rate": 4.087355496656321e-05, "loss": 0.0011658951407298446, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00117, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.5427979230880737, "learning_rate": 4.05982343699588e-05, "loss": 0.0030663476791232824, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00307, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.04047195613384247, "learning_rate": 4.0323513089607876e-05, "loss": 0.0008514574728906155, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00085, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.033537086099386215, "learning_rate": 4.004940255778431e-05, "loss": 0.0006391555070877075, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00064, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.0554814338684082, "learning_rate": 3.977591418134619e-05, "loss": 0.0008764395024627447, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00088, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.023144632577896118, "learning_rate": 3.95030593412612e-05, "loss": 0.0005167260533198714, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00052, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.38033297657966614, "learning_rate": 3.923084939213296e-05, "loss": 0.0036269566044211388, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00363, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.3443531394004822, "learning_rate": 3.895929566172861e-05, "loss": 0.005456274375319481, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00547, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.11662884056568146, "learning_rate": 3.868840945050728e-05, "loss": 0.0008762222132645547, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00088, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.49, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.10581538081169128, "learning_rate": 3.841820203114995e-05, "loss": 0.0007839235477149487, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00078, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.3213912546634674, "learning_rate": 3.814868464809027e-05, "loss": 0.0038530875463038683, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00386, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.2253940999507904, "learning_rate": 3.787986851704667e-05, "loss": 0.002273974707350135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00228, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.07796842604875565, "learning_rate": 3.7611764824555654e-05, "loss": 0.0009446886251680553, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00095, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.12933661043643951, "learning_rate": 3.734438472750619e-05, "loss": 0.0013748680939897895, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00138, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.1968342512845993, "learning_rate": 3.707773935267552e-05, "loss": 0.0022545757237821817, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00226, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.0040247756987810135, "learning_rate": 3.68118397962661e-05, "loss": 0.00010271165956510231, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.0001, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.01470676064491272, "learning_rate": 3.654669712344384e-05, "loss": 0.000204183510504663, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.0002, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.63, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.01582484506070614, "learning_rate": 3.628232236787763e-05, "loss": 0.00031882565235719085, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00032, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.63, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.007300813682377338, "learning_rate": 3.6018726531280144e-05, "loss": 9.944810881279409e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0001, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.5347946286201477, "learning_rate": 3.575592058295017e-05, "loss": 0.0017028674483299255, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0017, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.0037160846404731274, "learning_rate": 3.549391545931585e-05, "loss": 6.267199933063239e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00006, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.02053891122341156, "learning_rate": 3.5232722063479914e-05, "loss": 0.00010377274884376675, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0001, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.01682485267519951, "learning_rate": 3.49723512647657e-05, "loss": 0.0001953024184331298, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0002, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.03708474710583687, "learning_rate": 3.471281389826491e-05, "loss": 0.00028301539714448154, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00028, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.05116046592593193, "learning_rate": 3.4454120764386764e-05, "loss": 0.00037797511322423816, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00038, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.530811607837677, "learning_rate": 3.4196282628408526e-05, "loss": 0.008363794535398483, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0084, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.027868695557117462, "learning_rate": 3.3939310220027456e-05, "loss": 0.00027833806234411895, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00028, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.006396617740392685, "learning_rate": 3.3683214232914404e-05, "loss": 6.406758620869368e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00006, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.031217096373438835, "learning_rate": 3.342800532426873e-05, "loss": 0.00021793079213239253, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00022, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.009574182331562042, "learning_rate": 3.317369411437484e-05, "loss": 0.00012177053577033803, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00012, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.04631144925951958, "learning_rate": 3.292029118616024e-05, "loss": 0.00014955922961235046, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00015, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.21599379181861877, "learning_rate": 3.266780708475511e-05, "loss": 0.0003394159721210599, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00034, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.0056166937574744225, "learning_rate": 3.241625231705354e-05, "loss": 5.395221887738444e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00005, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.001352363615296781, "learning_rate": 3.216563735127618e-05, "loss": 2.6974799766321667e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00003, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.006542951334267855, "learning_rate": 3.191597261653475e-05, "loss": 5.029733438277617e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00005, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.2792622745037079, "learning_rate": 3.166726850239794e-05, "loss": 0.003004875499755144, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00301, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.0021990195382386446, "learning_rate": 3.141953535845912e-05, "loss": 4.7147885197773576e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00005, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.0033823768608272076, "learning_rate": 3.11727834939056e-05, "loss": 3.391222344362177e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00003, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.4654419720172882, "learning_rate": 3.092702317708967e-05, "loss": 0.009351884946227074, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0094, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.0019155471818521619, "learning_rate": 3.0682264635101276e-05, "loss": 3.883334284182638e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00004, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.11206567287445068, "learning_rate": 3.0438518053342407e-05, "loss": 0.0006789217004552484, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00068, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.02668733336031437, "learning_rate": 3.0195793575103266e-05, "loss": 0.00016038758622016758, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00016, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.24, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.2568329870700836, "learning_rate": 2.9954101301140146e-05, "loss": 0.001707063289359212, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00171, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.4870408773422241, "learning_rate": 2.9713451289255123e-05, "loss": 0.013284321874380112, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.01337, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.0023052028845995665, "learning_rate": 2.9473853553877484e-05, "loss": 5.0352944526821375e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00005, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.018688907846808434, "learning_rate": 2.9235318065647e-05, "loss": 0.0001958920038305223, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0002, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.31263628602027893, "learning_rate": 2.8997854750998964e-05, "loss": 0.0018437994876876473, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00185, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.010177414864301682, "learning_rate": 2.8761473491751258e-05, "loss": 0.00013179708912502974, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00013, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.2591210901737213, "learning_rate": 2.8526184124692883e-05, "loss": 0.002320799743756652, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00232, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.37, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.38457825779914856, "learning_rate": 2.829199644117484e-05, "loss": 0.0037369574420154095, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00374, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.42703133821487427, "learning_rate": 2.8058920186702553e-05, "loss": 0.004423916805535555, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00443, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.010350121185183525, "learning_rate": 2.782696506053033e-05, "loss": 9.874047100311145e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.0001, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.08575781434774399, "learning_rate": 2.7596140715257824e-05, "loss": 0.00040629052091389894, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00041, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.012914062477648258, "learning_rate": 2.7366456756428184e-05, "loss": 0.00014114304212853312, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00014, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.04061953350901604, "learning_rate": 2.7137922742128486e-05, "loss": 0.0003530512913130224, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00035, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.0316002182662487, "learning_rate": 2.691054818259188e-05, "loss": 0.00032838055631145835, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00033, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.04578730836510658, "learning_rate": 2.6684342539801933e-05, "loss": 0.0001978061773115769, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.0002, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.0981380045413971, "learning_rate": 2.645931522709877e-05, "loss": 0.0009938196744769812, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00099, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.021019259467720985, "learning_rate": 2.6235475608787365e-05, "loss": 0.0002570407814346254, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00026, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.09278926998376846, "learning_rate": 2.6012832999747916e-05, "loss": 0.0006888484349474311, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00069, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 1.1665046215057373, "learning_rate": 2.579139666504821e-05, "loss": 0.017750291153788567, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01791, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.04, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.09103650599718094, "learning_rate": 2.557117581955798e-05, "loss": 0.0004786623176187277, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00048, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.34270042181015015, "learning_rate": 2.5352179627565532e-05, "loss": 0.005320657975971699, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00533, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.0200478695333004, "learning_rate": 2.5134417202396277e-05, "loss": 0.00016091702855192125, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00016, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.2225683629512787, "learning_rate": 2.491789760603361e-05, "loss": 0.001919064437970519, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00192, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.01377931609749794, "learning_rate": 2.4702629848741764e-05, "loss": 0.00015642112703062594, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00016, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.010027467273175716, "learning_rate": 2.4488622888690785e-05, "loss": 0.00013296204269863665, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00013, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.0033969960641115904, "learning_rate": 2.427588563158384e-05, "loss": 7.989332516444847e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00008, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.43507349491119385, "learning_rate": 2.406442693028651e-05, "loss": 0.002374065574258566, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00238, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.05506065487861633, "learning_rate": 2.3854255584458547e-05, "loss": 0.0005187464994378388, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00052, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.2068173587322235, "learning_rate": 2.3645380340187508e-05, "loss": 0.001083013485185802, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00108, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.06186562404036522, "learning_rate": 2.3437809889624914e-05, "loss": 0.000740391609724611, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00074, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.15988218784332275, "learning_rate": 2.3231552870624487e-05, "loss": 0.0016715346137061715, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00167, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.14, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.012994193471968174, "learning_rate": 2.3026617866382657e-05, "loss": 0.0001652940409258008, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00017, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.02667989954352379, "learning_rate": 2.2823013405081507e-05, "loss": 0.00030424160649999976, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0003, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.3329944312572479, "learning_rate": 2.2620747959533722e-05, "loss": 0.00152586807962507, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00153, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.64, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.006472993176430464, "learning_rate": 2.2419829946830123e-05, "loss": 0.00011014008487109095, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00011, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.022732587531208992, "learning_rate": 2.2220267727989325e-05, "loss": 0.00010125887638423592, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0001, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.2217869609594345, "learning_rate": 2.202206960760984e-05, "loss": 0.0032647503539919853, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00327, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.05782803148031235, "learning_rate": 2.182524383352446e-05, "loss": 0.0005775754107162356, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00058, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.27409785985946655, "learning_rate": 2.1629798596457056e-05, "loss": 0.013317692093551159, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01341, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.014245536178350449, "learning_rate": 2.1435742029681725e-05, "loss": 0.00016649045574013144, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00017, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.022492293268442154, "learning_rate": 2.124308220868431e-05, "loss": 0.000158926093718037, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00016, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.38484513759613037, "learning_rate": 2.105182715082638e-05, "loss": 0.0017742939526215196, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00178, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.17856812477111816, "learning_rate": 2.0861984815011552e-05, "loss": 0.0031276647932827473, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00313, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.00514964247122407, "learning_rate": 2.0673563101354323e-05, "loss": 0.00010401933832326904, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0001, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.17853279411792755, "learning_rate": 2.0486569850851317e-05, "loss": 0.001258289790712297, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00126, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.009071777574717999, "learning_rate": 2.0301012845054956e-05, "loss": 0.00017314331489615142, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00017, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.51, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.005737656727433205, "learning_rate": 2.011689980574966e-05, "loss": 9.981922630686313e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.0001, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.16713295876979828, "learning_rate": 1.993423839463052e-05, "loss": 0.0023319122847169638, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00233, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.89, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.012096023187041283, "learning_rate": 1.975303621298445e-05, "loss": 9.533599950373173e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0001, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.00686148414388299, "learning_rate": 1.957330080137385e-05, "loss": 0.00012745779531542212, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00013, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.3535265028476715, "learning_rate": 1.9395039639322864e-05, "loss": 0.002871020929887891, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00288, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.0015710110310465097, "learning_rate": 1.9218260145006073e-05, "loss": 3.912924148607999e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00004, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.034198787063360214, "learning_rate": 1.904296967493982e-05, "loss": 0.00022704925504513085, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00023, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.06529953330755234, "learning_rate": 1.8869175523676064e-05, "loss": 0.0008366560214199126, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00084, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.004674799740314484, "learning_rate": 1.869688492349885e-05, "loss": 5.897530718357302e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00006, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.014150934293866158, "learning_rate": 1.85261050441233e-05, "loss": 0.00020906679856125265, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00021, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.43929311633110046, "learning_rate": 1.8356842992397304e-05, "loss": 0.009146887809038162, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00919, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.010821101255714893, "learning_rate": 1.8189105812005714e-05, "loss": 0.00019259407417848706, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00019, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.022763732820749283, "learning_rate": 1.802290048317732e-05, "loss": 0.00023526756558567286, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00024, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.019374454393982887, "learning_rate": 1.785823392239424e-05, "loss": 0.00021382395061664283, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00021, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.11978479474782944, "learning_rate": 1.7695112982104225e-05, "loss": 0.0013092365115880966, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00131, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.008423391729593277, "learning_rate": 1.7533544450435433e-05, "loss": 0.0001229287008754909, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00012, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.07, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.013656062074005604, "learning_rate": 1.7373535050913946e-05, "loss": 0.00015506724594160914, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00016, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.89, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.35753321647644043, "learning_rate": 1.721509144218405e-05, "loss": 0.003023238619789481, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00303, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.003856342053040862, "learning_rate": 1.705822021773101e-05, "loss": 5.484955181600526e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00005, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.1869206577539444, "learning_rate": 1.69029279056068e-05, "loss": 0.005816401448100805, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00583, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.08508329838514328, "learning_rate": 1.6749220968158415e-05, "loss": 0.00045704032527282834, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00046, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.06945516169071198, "learning_rate": 1.659710580175893e-05, "loss": 0.0008879072847776115, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00089, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.12, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.04523242637515068, "learning_rate": 1.644658873654133e-05, "loss": 0.0004053134471178055, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00041, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.008180685341358185, "learning_rate": 1.629767603613508e-05, "loss": 9.613325528334826e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.0001, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.24292775988578796, "learning_rate": 1.615037389740547e-05, "loss": 0.0038110159803181887, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00382, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.028058597818017006, "learning_rate": 1.600468845019576e-05, "loss": 0.0002432062174193561, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00024, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.01821316033601761, "learning_rate": 1.5860625757072092e-05, "loss": 8.466571307508275e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00008, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.009544719010591507, "learning_rate": 1.571819181307116e-05, "loss": 9.454159589949995e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00009, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.12010902166366577, "learning_rate": 1.557739254545075e-05, "loss": 0.0006743200356140733, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00067, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.7, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.0030458965338766575, "learning_rate": 1.543823381344311e-05, "loss": 6.628083065152168e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00007, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.16, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.014733631163835526, "learning_rate": 1.5300721408011114e-05, "loss": 0.0001961405505426228, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.0002, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.008656126447021961, "learning_rate": 1.5164861051607254e-05, "loss": 0.00015997247828636318, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00016, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.004855229053646326, "learning_rate": 1.5030658397935521e-05, "loss": 3.68240725947544e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00004, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.05417121946811676, "learning_rate": 1.4898119031716104e-05, "loss": 0.0006794093060307205, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00068, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.19294682145118713, "learning_rate": 1.476724846845306e-05, "loss": 0.0005086844903416932, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00051, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.35, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.03232863172888756, "learning_rate": 1.463805215420471e-05, "loss": 0.0004260529822204262, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00043, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.016846485435962677, "learning_rate": 1.451053546535705e-05, "loss": 0.0001866669044829905, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00019, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.005087986588478088, "learning_rate": 1.438470370840001e-05, "loss": 8.697484736330807e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00009, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.1141049861907959, "learning_rate": 1.4260562119706606e-05, "loss": 0.0010637581581249833, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00106, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.01379898190498352, "learning_rate": 1.413811586531508e-05, "loss": 0.0002449744497425854, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00025, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.10184764117002487, "learning_rate": 1.4017370040713884e-05, "loss": 0.0007206881418824196, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00072, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.006620363332331181, "learning_rate": 1.3898329670629645e-05, "loss": 0.00011032049951609224, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00011, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.99, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.05515965074300766, "learning_rate": 1.3780999708818058e-05, "loss": 0.00036459576222114265, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00036, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.54, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.0025299040134996176, "learning_rate": 1.3665385037857758e-05, "loss": 4.405796425999142e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00004, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.23209916055202484, "learning_rate": 1.3551490468947126e-05, "loss": 0.0013016184093430638, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.0013, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.24373836815357208, "learning_rate": 1.3439320741704075e-05, "loss": 0.002676041331142187, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00268, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.04362897947430611, "learning_rate": 1.3328880523968808e-05, "loss": 0.0004681990831159055, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00047, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.06471847742795944, "learning_rate": 1.3220174411609587e-05, "loss": 0.0003086658543907106, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00031, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.04582749679684639, "learning_rate": 1.3113206928331471e-05, "loss": 0.0003662576200440526, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00037, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.76, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.09040702879428864, "learning_rate": 1.300798252548806e-05, "loss": 0.0005248327506706119, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00052, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.4299568235874176, "learning_rate": 1.2904505581896265e-05, "loss": 0.005051751621067524, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00506, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.010928449220955372, "learning_rate": 1.2802780403654082e-05, "loss": 0.00014635160914622247, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00015, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.11942346394062042, "learning_rate": 1.2702811223961408e-05, "loss": 0.001370937330648303, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00137, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.014610650949180126, "learning_rate": 1.2604602202943861e-05, "loss": 0.00013475684681907296, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00013, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.13283032178878784, "learning_rate": 1.2508157427479686e-05, "loss": 0.0008815507753752172, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00088, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.10970672965049744, "learning_rate": 1.2413480911029655e-05, "loss": 0.00017873088654596359, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00018, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.010239587165415287, "learning_rate": 1.2320576593470082e-05, "loss": 8.68424103828147e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00009, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.2813096344470978, "learning_rate": 1.2229448340928828e-05, "loss": 0.0016313226660713553, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00163, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.010883024893701077, "learning_rate": 1.2140099945624458e-05, "loss": 0.00014181036385707557, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00014, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.019178101792931557, "learning_rate": 1.205253512570841e-05, "loss": 0.00020821092766709626, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00021, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.0104290172457695, "learning_rate": 1.1966757525110255e-05, "loss": 0.00010148670116905123, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.0001, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.4827379286289215, "learning_rate": 1.1882770713386095e-05, "loss": 0.003550226567313075, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00356, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.003940227907150984, "learning_rate": 1.180057818556998e-05, "loss": 4.908836126560345e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00005, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.002890810137614608, "learning_rate": 1.1720183362028494e-05, "loss": 5.593815876636654e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00006, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.9874055981636047, "learning_rate": 1.1641589588318387e-05, "loss": 0.0027585469651967287, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00276, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.035522907972335815, "learning_rate": 1.1564800135047418e-05, "loss": 8.013202750589699e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00008, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.0017549411859363317, "learning_rate": 1.148981819773816e-05, "loss": 3.0141693059704266e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00003, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.15154358744621277, "learning_rate": 1.1416646896695086e-05, "loss": 0.0012299786321818829, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00123, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.1740664541721344, "learning_rate": 1.1345289276874717e-05, "loss": 0.0011553571093827486, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00116, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.0035063426475971937, "learning_rate": 1.1275748307758873e-05, "loss": 5.1999704737681895e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00005, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.21438570320606232, "learning_rate": 1.1208026883231147e-05, "loss": 0.0008126706816256046, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00081, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.0668414756655693, "learning_rate": 1.1142127821456433e-05, "loss": 0.00019812912796624005, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0002, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.02570449188351631, "learning_rate": 1.1078053864763674e-05, "loss": 0.00018047240155283362, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00018, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.1000296100974083, "learning_rate": 1.1015807679531756e-05, "loss": 0.0005478369421325624, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00055, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.06822922080755234, "learning_rate": 1.0955391856078528e-05, "loss": 0.00036245150840841234, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00036, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.2640154957771301, "learning_rate": 1.0896808908553007e-05, "loss": 0.00172343326266855, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00172, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.07862810045480728, "learning_rate": 1.0840061274830763e-05, "loss": 0.00022179083316586912, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00022, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.006681669969111681, "learning_rate": 1.0785151316412473e-05, "loss": 8.200207230402157e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00008, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.006688909139484167, "learning_rate": 1.0732081318325639e-05, "loss": 7.407966768369079e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00007, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.2600274384021759, "learning_rate": 1.0680853489029501e-05, "loss": 0.0016795732080936432, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00168, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.01315993256866932, "learning_rate": 1.0631469960323152e-05, "loss": 0.0001818336604628712, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00018, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.59, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.15525273978710175, "learning_rate": 1.0583932787256783e-05, "loss": 0.0013891574926674366, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00139, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.02032255381345749, "learning_rate": 1.0538243948046206e-05, "loss": 0.0001492321607656777, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00015, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.003387365723028779, "learning_rate": 1.0494405343990523e-05, "loss": 4.830621401197277e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00005, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.07823483645915985, "learning_rate": 1.0452418799392985e-05, "loss": 0.0005165119073353708, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00052, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.299793541431427, "learning_rate": 1.0412286061485102e-05, "loss": 0.010191489942371845, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.01024, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0010845506330952048, "learning_rate": 1.03740088003539e-05, "loss": 2.206624594691675e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00002, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.61, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.0026962896808981895, "learning_rate": 1.0337588608872463e-05, "loss": 4.785431519849226e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00005, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.09, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.7083113193511963, "learning_rate": 1.0303027002633622e-05, "loss": 0.006875107996165752, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0069, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.007653111591935158, "learning_rate": 1.0270325419886884e-05, "loss": 8.413799514528364e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00008, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.7313287854194641, "learning_rate": 1.0239485221478599e-05, "loss": 0.0006137129385024309, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00061, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.005833817645907402, "learning_rate": 1.0210507690795292e-05, "loss": 3.626795660238713e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00004, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.0015502205351367593, "learning_rate": 1.0183394033710305e-05, "loss": 3.313478737254627e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00003, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.02749207615852356, "learning_rate": 1.0158145378533583e-05, "loss": 0.000305008637951687, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00031, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.26, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.01304501760751009, "learning_rate": 1.0134762775964726e-05, "loss": 0.0001247023610631004, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00012, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.019421285018324852, "learning_rate": 1.0113247199049278e-05, "loss": 0.00015739737136755139, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00016, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.12174617499113083, "learning_rate": 1.0093599543138205e-05, "loss": 0.0008953351061791182, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.0009, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.5543283820152283, "learning_rate": 1.0075820625850675e-05, "loss": 0.006211260333657265, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00623, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.004298795480281115, "learning_rate": 1.0059911187040013e-05, "loss": 9.000624413602054e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00009, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.005462809931486845, "learning_rate": 1.0045871888762893e-05, "loss": 8.601414447184652e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00009, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.035554055124521255, "learning_rate": 1.003370331525184e-05, "loss": 0.0002683405764400959, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00027, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.002631516195833683, "learning_rate": 1.002340597289085e-05, "loss": 3.118737004115246e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00003, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.006511299405246973, "learning_rate": 1.0014980290194387e-05, "loss": 8.244947821367532e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00008, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.054458409547805786, "learning_rate": 1.0008426617789489e-05, "loss": 0.0004670056514441967, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00047, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.009516173973679543, "learning_rate": 1.0003745228401215e-05, "loss": 9.874042007140815e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0001, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.055273622274398804, "learning_rate": 1.0000936316841296e-05, "loss": 0.0002525469462852925, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00025, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.98, "tokens/trainable": 232756} diff --git a/gemma3_12b_19m/control/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_19m/control/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cedd6b04903e7d7b026547b01059cd293683e3a3 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_19m/control/training/aft/mixed_coin/train.log b/gemma3_12b_19m/control/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..86255562d5f7423c973f8e2ce1224ae5d723ed70 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:677: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-02 07:51:32,647] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:51:34.707000 188959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:51:34.730000 188959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 07:51:44,683] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 07:51:47.570000 190007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:51:47.591000 190007 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 07:51:50,079] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:51:50,081] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 07:51:50,081] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:51:50,104] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 07:51:50,159] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 07:51:50,159] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-02 07:51:50,246] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-02 07:51:52,939] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/prepared/89a3673a7a5d75006d308d5ef27c29a7 +[2026-09-02 07:51:52,939] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-02 07:51:52,939] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 32992.35 examples/s] Generating train split: 8002 examples [00:00, 29743.09 examples/s] Generating train split: 8192 examples [00:00, 29594.26 examples/s] +[2026-09-02 07:51:54,245] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-02 07:51:54,247] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 5534.95 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 24700.42 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:48.218000 190721 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:48.925000 190720 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:48.947000 190720 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:48.948000 190719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:48.948000 190725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:48.970000 190719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:48.970000 190725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.021000 190723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.043000 190723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.044000 190726 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.066000 190726 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.077000 190722 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.099000 190722 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.122000 190730 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.122000 190724 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.144000 190730 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 07:52:49.145000 190724 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:11, 100.79 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:11, 100.79 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:00, 100.79 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:50, 100.79 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:40, 100.79 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:30, 100.79 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:20, 100.79 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 100.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 100.79 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 725.86 examples/s] +[2026-09-02 07:52:53,603] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-02 07:53:02,771] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-02T09:08:22+00:00", + "resolved_config_sha256": "f2d78850839c6ddbeba2bd9ed292f2076221a5db16989061d1ad846f1969321a", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "97776cd3416926ab6bbd46577f9a2c255c891f1a0411e6acb1d29816771e9657", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/control/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "b8ccc9dd1ff6d1a5d4c9836b2f1306c5585d18b928999f16b9ff0a7b786e7394", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_19m/control/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_19m/control/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..84d169b7c7bc19ef2fd6a2738243d81c86792db1 --- /dev/null +++ b/gemma3_12b_19m/control/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.1124460697174072, "learning_rate": 0.0, "loss": 0.2915830910205841, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.33854, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 3.4324753284454346, "learning_rate": 4.000000000000001e-06, "loss": 0.28511565923690796, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.32992, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.69, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 3.2998151779174805, "learning_rate": 8.000000000000001e-06, "loss": 0.27595555782318115, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.31779, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 3.290576934814453, "learning_rate": 1.2e-05, "loss": 0.2665245532989502, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.30542, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.57, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 2.910008430480957, "learning_rate": 1.6000000000000003e-05, "loss": 0.24833008646965027, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.28188, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 2.112990140914917, "learning_rate": 2e-05, "loss": 0.2053617238998413, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.22797, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.5289608240127563, "learning_rate": 2.4e-05, "loss": 0.18311187624931335, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.20095, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.416357159614563, "learning_rate": 2.8000000000000003e-05, "loss": 0.14626747369766235, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.15751, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 2.0328433513641357, "learning_rate": 3.2000000000000005e-05, "loss": 0.11774111539125443, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.12495, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.8074275255203247, "learning_rate": 3.6e-05, "loss": 0.13423021137714386, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.14366, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.6083096265792847, "learning_rate": 4e-05, "loss": 0.09807267040014267, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10304, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 4.797353267669678, "learning_rate": 4.4000000000000006e-05, "loss": 0.11410786211490631, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.12087, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 1.9674979448318481, "learning_rate": 4.8e-05, "loss": 0.08029379695653915, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.08361, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 1.740987777709961, "learning_rate": 5.2000000000000004e-05, "loss": 0.08639811724424362, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.09024, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 5.38066291809082, "learning_rate": 5.6000000000000006e-05, "loss": 0.10535287111997604, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.1111, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 4.236367702484131, "learning_rate": 6e-05, "loss": 0.0835435763001442, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.08713, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 13.871305465698242, "learning_rate": 6.400000000000001e-05, "loss": 0.05163346976041794, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.05299, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 32.117469787597656, "learning_rate": 6.800000000000001e-05, "loss": 0.0510420948266983, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05237, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 1.3785450458526611, "learning_rate": 7.2e-05, "loss": 0.05737077444791794, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.05905, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 1.6824352741241455, "learning_rate": 7.6e-05, "loss": 0.06822812557220459, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.07061, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.1043897867202759, "learning_rate": 8e-05, "loss": 0.03987414389848709, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.04068, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 5.495035648345947, "learning_rate": 8.4e-05, "loss": 0.03167415410280228, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03218, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.9126105308532715, "learning_rate": 8.800000000000001e-05, "loss": 0.04192593693733215, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.04282, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.35, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.9045581817626953, "learning_rate": 9.200000000000001e-05, "loss": 0.03636747598648071, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.03704, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.99, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.7142493724823, "learning_rate": 9.6e-05, "loss": 0.06277583539485931, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.06479, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 6.057705879211426, "learning_rate": 0.0001, "loss": 0.06420055031776428, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.06631, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 3.5359790325164795, "learning_rate": 9.99990636831587e-05, "loss": 0.051677849143743515, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.05304, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.9098083972930908, "learning_rate": 9.999625477159879e-05, "loss": 0.05589593946933746, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.05749, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.7376760244369507, "learning_rate": 9.999157338221051e-05, "loss": 0.031809061765670776, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03232, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.4985719919204712, "learning_rate": 9.998501970980562e-05, "loss": 0.039047908037900925, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.03982, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.39, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.6362605094909668, "learning_rate": 9.997659402710915e-05, "loss": 0.02248135767877102, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02274, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.0491445064544678, "learning_rate": 9.996629668474818e-05, "loss": 0.042541682720184326, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04346, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.65, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.7631808519363403, "learning_rate": 9.995412811123711e-05, "loss": 0.02031688392162323, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02052, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 1.3441791534423828, "learning_rate": 9.994008881295999e-05, "loss": 0.030500896275043488, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.03097, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.7924111485481262, "learning_rate": 9.992417937414932e-05, "loss": 0.02520655281841755, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02553, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.93, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.3451659679412842, "learning_rate": 9.99064004568618e-05, "loss": 0.010555921122431755, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01061, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.12391197681427, "learning_rate": 9.988675280095074e-05, "loss": 0.02493421360850334, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02525, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.34112548828125, "learning_rate": 9.986523722403528e-05, "loss": 0.016177719458937645, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01631, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.2314828634262085, "learning_rate": 9.984185462146642e-05, "loss": 0.024499662220478058, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.0248, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.8459354639053345, "learning_rate": 9.98166059662897e-05, "loss": 0.019456809386610985, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01965, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.5, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.7103182673454285, "learning_rate": 9.978949230920472e-05, "loss": 0.012013943865895271, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01209, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.3468751907348633, "learning_rate": 9.976051477852141e-05, "loss": 0.05897052586078644, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.06074, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 2.0088162422180176, "learning_rate": 9.972967458011312e-05, "loss": 0.07060430198907852, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.07316, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.2542554140090942, "learning_rate": 9.96969729973664e-05, "loss": 0.048635661602020264, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.04984, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.5752454400062561, "learning_rate": 9.966241139112754e-05, "loss": 0.01791270263493061, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01807, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.834753155708313, "learning_rate": 9.96259911996461e-05, "loss": 0.02946607582271099, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.0299, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 1.0974671840667725, "learning_rate": 9.958771393851491e-05, "loss": 0.05380699783563614, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.05528, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 3.7793161869049072, "learning_rate": 9.954758120060702e-05, "loss": 0.0320456326007843, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03256, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.9503911733627319, "learning_rate": 9.950559465600948e-05, "loss": 0.025890083983540535, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.02623, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.5428978800773621, "learning_rate": 9.946175605195379e-05, "loss": 0.02979426085948944, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.03024, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.3681890666484833, "learning_rate": 9.941606721274322e-05, "loss": 0.009715531021356583, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.00976, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.8235594034194946, "learning_rate": 9.936853003967685e-05, "loss": 0.03638250008225441, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03705, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.1632903814315796, "learning_rate": 9.93191465109705e-05, "loss": 0.029520178213715553, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.02996, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 7.265408992767334, "learning_rate": 9.926791868167438e-05, "loss": 0.040746744722127914, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.04159, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.9152982234954834, "learning_rate": 9.921484868358753e-05, "loss": 0.06606098264455795, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.06829, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.5971580743789673, "learning_rate": 9.915993872516924e-05, "loss": 0.01729523576796055, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01745, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.3977312445640564, "learning_rate": 9.9103191091447e-05, "loss": 0.011339948512613773, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0114, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.37808653712272644, "learning_rate": 9.904460814392147e-05, "loss": 0.01223925780504942, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01231, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.8054453134536743, "learning_rate": 9.898419232046825e-05, "loss": 0.03150083124637604, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.032, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.9487889409065247, "learning_rate": 9.892194613523633e-05, "loss": 0.03553062304854393, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.03617, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.43623286485671997, "learning_rate": 9.885787217854357e-05, "loss": 0.02163667604327202, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.02187, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.47369375824928284, "learning_rate": 9.879197311676887e-05, "loss": 0.022778213024139404, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02304, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.8295040130615234, "learning_rate": 9.872425169224113e-05, "loss": 0.034510452300310135, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.03511, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.41608715057373047, "learning_rate": 9.865471072312528e-05, "loss": 0.023070182651281357, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02334, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.3821655809879303, "learning_rate": 9.858335310330492e-05, "loss": 0.013968494720757008, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01407, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.3925122916698456, "learning_rate": 9.851018180226185e-05, "loss": 0.01750391162931919, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01766, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.52, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.6867932677268982, "learning_rate": 9.843519986495259e-05, "loss": 0.01603938825428486, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01617, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.5391637682914734, "learning_rate": 9.835841041168162e-05, "loss": 0.02560647390782833, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02594, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 1.415567398071289, "learning_rate": 9.82798166379715e-05, "loss": 0.020187923684716225, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02039, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.5755669474601746, "learning_rate": 9.819942181443002e-05, "loss": 0.024706564843654633, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.02501, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.6135363578796387, "learning_rate": 9.811722928661392e-05, "loss": 0.015225745737552643, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.01534, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.44252175092697144, "learning_rate": 9.803324247488975e-05, "loss": 0.008068669587373734, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0081, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.5863214731216431, "learning_rate": 9.794746487429161e-05, "loss": 0.010686604306101799, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01074, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.2188020944595337, "learning_rate": 9.785990005437554e-05, "loss": 0.00565185584127903, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00567, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.89292311668396, "learning_rate": 9.777055165907117e-05, "loss": 0.02014061249792576, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.02034, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.5671249032020569, "learning_rate": 9.767942340652993e-05, "loss": 0.025112072005867958, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02543, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.3694917857646942, "learning_rate": 9.758651908897035e-05, "loss": 0.011164989322423935, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01123, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 0.5070821046829224, "learning_rate": 9.749184257252033e-05, "loss": 0.016489244997501373, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.01663, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.4839690327644348, "learning_rate": 9.739539779705614e-05, "loss": 0.013143117539584637, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01323, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.57, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.3991265892982483, "learning_rate": 9.729718877603861e-05, "loss": 0.013821924105286598, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01392, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 1.1962260007858276, "learning_rate": 9.719721959634592e-05, "loss": 0.04509039223194122, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.04612, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.2641296982765198, "learning_rate": 9.709549441810375e-05, "loss": 0.006131240166723728, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00615, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.5739407539367676, "learning_rate": 9.699201747451195e-05, "loss": 0.015413038432598114, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01553, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.2321222722530365, "learning_rate": 9.688679307166854e-05, "loss": 0.005475499201565981, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00549, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.72414630651474, "learning_rate": 9.677982558839042e-05, "loss": 0.030317291617393494, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.03078, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.40501734614372253, "learning_rate": 9.66711194760312e-05, "loss": 0.01103794015944004, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0111, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.6381338238716125, "learning_rate": 9.656067925829593e-05, "loss": 0.014392191544175148, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0145, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.34184834361076355, "learning_rate": 9.644850953105288e-05, "loss": 0.015225250273942947, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.01534, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.3202052414417267, "learning_rate": 9.633461496214225e-05, "loss": 0.017231786623597145, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01738, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.5138589143753052, "learning_rate": 9.621900029118195e-05, "loss": 0.04720668867230415, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.04834, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.5888076424598694, "learning_rate": 9.610167032937036e-05, "loss": 0.010753209702670574, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01081, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.24163059890270233, "learning_rate": 9.598262995928611e-05, "loss": 0.015943225473165512, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01607, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.2696418762207031, "learning_rate": 9.586188413468492e-05, "loss": 0.004883126821368933, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0049, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.3414899706840515, "learning_rate": 9.57394378802934e-05, "loss": 0.017462214455008507, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01762, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.7119160294532776, "learning_rate": 9.56152962916e-05, "loss": 0.024609733372926712, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.02492, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.13914279639720917, "learning_rate": 9.548946453464296e-05, "loss": 0.0022393749095499516, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00224, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.258314311504364, "learning_rate": 9.53619478457953e-05, "loss": 0.010911088436841965, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01097, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.63115394115448, "learning_rate": 9.523275153154695e-05, "loss": 0.007371758110821247, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.0074, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.48338407278060913, "learning_rate": 9.51018809682839e-05, "loss": 0.014411164447665215, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01452, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.8213213086128235, "learning_rate": 9.49693416020645e-05, "loss": 0.033086881041526794, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.03364, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.1377916932106018, "learning_rate": 9.483513894839276e-05, "loss": 0.002307554241269827, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00231, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.8331613540649414, "learning_rate": 9.469927859198888e-05, "loss": 0.024406515061855316, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02471, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.6150726079940796, "learning_rate": 9.456176618655689e-05, "loss": 0.0290459506213665, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02947, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 0.4791085422039032, "learning_rate": 9.442260745454927e-05, "loss": 0.00835009291768074, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00839, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.49311187863349915, "learning_rate": 9.428180818692884e-05, "loss": 0.0055901529267430305, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00561, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.298886239528656, "learning_rate": 9.413937424292791e-05, "loss": 0.012263728305697441, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01234, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.5907858610153198, "learning_rate": 9.399531154980424e-05, "loss": 0.020520389080047607, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.02073, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.2852950394153595, "learning_rate": 9.384962610259455e-05, "loss": 0.009021681733429432, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00906, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.2912001609802246, "learning_rate": 9.370232396386494e-05, "loss": 0.010330094955861568, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.01038, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.3081084191799164, "learning_rate": 9.355341126345868e-05, "loss": 0.011707223951816559, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01178, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.19509750604629517, "learning_rate": 9.340289419824107e-05, "loss": 0.008028751239180565, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00806, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.24366040527820587, "learning_rate": 9.325077903184159e-05, "loss": 0.005756698548793793, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00577, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.16715732216835022, "learning_rate": 9.30970720943932e-05, "loss": 0.004204008728265762, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00421, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.42864853143692017, "learning_rate": 9.2941779782269e-05, "loss": 0.006051333621144295, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00607, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.051573533564805984, "learning_rate": 9.278490855781596e-05, "loss": 0.000667284766677767, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00067, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 12.784049034118652, "learning_rate": 9.262646494908604e-05, "loss": 0.004714064300060272, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00473, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.4247305691242218, "learning_rate": 9.246645554956457e-05, "loss": 0.004983646795153618, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.005, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 0.7745587825775146, "learning_rate": 9.230488701789578e-05, "loss": 0.023289786651730537, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02356, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.6664260029792786, "learning_rate": 9.214176607760577e-05, "loss": 0.009867158718407154, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00992, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.8453553318977356, "learning_rate": 9.197709951682268e-05, "loss": 0.01721338741481304, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01736, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 29.04, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 1.6087898015975952, "learning_rate": 9.181089418799428e-05, "loss": 0.06910032778978348, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.07154, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.22566111385822296, "learning_rate": 9.164315700760271e-05, "loss": 0.0031619779765605927, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00317, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 1.018997311592102, "learning_rate": 9.147389495587671e-05, "loss": 0.011721212416887283, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01179, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.94, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.8730391263961792, "learning_rate": 9.130311507650116e-05, "loss": 0.02832825668156147, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.02873, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 1.3386648893356323, "learning_rate": 9.113082447632394e-05, "loss": 0.027853025123476982, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.02824, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.95, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.5551685094833374, "learning_rate": 9.09570303250602e-05, "loss": 0.023217901587486267, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.02349, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.4, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.957292377948761, "learning_rate": 9.078173985499394e-05, "loss": 0.015208153054118156, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01532, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.3337922990322113, "learning_rate": 9.060496036067713e-05, "loss": 0.015398605726659298, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01552, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 0.21708877384662628, "learning_rate": 9.042669919862615e-05, "loss": 0.010526532307267189, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01058, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.41966405510902405, "learning_rate": 9.024696378701557e-05, "loss": 0.018143344670534134, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01831, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.508224368095398, "learning_rate": 9.006576160536948e-05, "loss": 0.017805572599172592, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01797, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.3274213671684265, "learning_rate": 8.988310019425035e-05, "loss": 0.008574122563004494, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00861, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.20282338559627533, "learning_rate": 8.969898715494506e-05, "loss": 0.006176254712045193, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0062, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 1.050326943397522, "learning_rate": 8.951343014914869e-05, "loss": 0.02194802090525627, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02219, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.08848333358764648, "learning_rate": 8.932643689864568e-05, "loss": 0.0011272112606093287, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00113, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.18943892419338226, "learning_rate": 8.913801518498845e-05, "loss": 0.0026484006084501743, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00265, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.16, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.1014225110411644, "learning_rate": 8.894817284917364e-05, "loss": 0.000776435830630362, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00078, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.8198901414871216, "learning_rate": 8.875691779131569e-05, "loss": 0.01870388723909855, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01888, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.6, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.18202698230743408, "learning_rate": 8.856425797031829e-05, "loss": 0.0005617749411612749, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00056, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 0.6323541402816772, "learning_rate": 8.837020140354295e-05, "loss": 0.0076285237446427345, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00766, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.1938227266073227, "learning_rate": 8.817475616647554e-05, "loss": 0.0014366444665938616, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00144, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.5044165849685669, "learning_rate": 8.797793039239017e-05, "loss": 0.0014520962722599506, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00145, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.9513782858848572, "learning_rate": 8.777973227201069e-05, "loss": 0.016751009970903397, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01689, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.60610431432724, "learning_rate": 8.758017005316988e-05, "loss": 0.013404827564954758, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.0135, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.5665386915206909, "learning_rate": 8.737925204046629e-05, "loss": 0.008844960480928421, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00888, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.40403565764427185, "learning_rate": 8.717698659491851e-05, "loss": 0.0022581557277590036, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00226, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.4799209535121918, "learning_rate": 8.697338213361735e-05, "loss": 0.02058997191488743, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.0208, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.39362430572509766, "learning_rate": 8.676844712937552e-05, "loss": 0.006884434726089239, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00691, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.6991528272628784, "learning_rate": 8.656219011037509e-05, "loss": 0.014200648292899132, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0143, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.09726765751838684, "learning_rate": 8.63546196598125e-05, "loss": 0.0011699337046593428, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00117, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.6196016669273376, "learning_rate": 8.614574441554145e-05, "loss": 0.020273998379707336, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.02048, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.2294650375843048, "learning_rate": 8.593557306971349e-05, "loss": 0.007050707936286926, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00708, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.408464252948761, "learning_rate": 8.572411436841618e-05, "loss": 0.005010885186493397, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00502, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.23371358215808868, "learning_rate": 8.551137711130922e-05, "loss": 0.008169584907591343, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.0082, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.24119523167610168, "learning_rate": 8.529737015125824e-05, "loss": 0.00614047609269619, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00616, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 1.2798469066619873, "learning_rate": 8.508210239396639e-05, "loss": 0.018541209399700165, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01871, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.226275235414505, "learning_rate": 8.486558279760375e-05, "loss": 0.005290142726153135, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.0053, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.25127190351486206, "learning_rate": 8.464782037243449e-05, "loss": 0.007989948615431786, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00802, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 1.0434913635253906, "learning_rate": 8.442882418044202e-05, "loss": 0.00756283663213253, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00759, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 29.04, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.6660630106925964, "learning_rate": 8.420860333495179e-05, "loss": 0.013208652846515179, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.0133, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.14531056582927704, "learning_rate": 8.398716700025208e-05, "loss": 0.003343078540638089, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00335, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.5779041051864624, "learning_rate": 8.376452439121266e-05, "loss": 0.017291821539402008, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01744, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.4179789125919342, "learning_rate": 8.354068477290124e-05, "loss": 0.009831881150603294, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00988, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.33168771862983704, "learning_rate": 8.331565746019807e-05, "loss": 0.006892091128975153, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00692, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.3348756432533264, "learning_rate": 8.308945181740812e-05, "loss": 0.005851908586919308, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00587, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.3425787687301636, "learning_rate": 8.286207725787153e-05, "loss": 0.006670833565294743, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00669, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.8465215563774109, "learning_rate": 8.263354324357182e-05, "loss": 0.010778047144412994, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01084, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.16291052103042603, "learning_rate": 8.240385928474219e-05, "loss": 0.004049909766763449, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00406, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.10040465742349625, "learning_rate": 8.217303493946967e-05, "loss": 0.0008951411582529545, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.0009, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.4308980405330658, "learning_rate": 8.194107981329746e-05, "loss": 0.014934166334569454, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01505, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.009303738363087177, "learning_rate": 8.170800355882518e-05, "loss": 0.0001444478693883866, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00014, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.27, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.969424307346344, "learning_rate": 8.147381587530713e-05, "loss": 0.01700567826628685, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01715, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 0.8431085348129272, "learning_rate": 8.123852650824877e-05, "loss": 0.007255835458636284, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00728, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.3206964135169983, "learning_rate": 8.100214524900103e-05, "loss": 0.005919390358030796, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00594, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.11, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.040951039642095566, "learning_rate": 8.076468193435301e-05, "loss": 0.0004116488271392882, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00041, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.5416037440299988, "learning_rate": 8.052614644612253e-05, "loss": 0.01502454187721014, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01514, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.36150437593460083, "learning_rate": 8.028654871074489e-05, "loss": 0.006234110798686743, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00625, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.7202771902084351, "learning_rate": 8.004589869885986e-05, "loss": 0.017932897433638573, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.01809, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.1422046422958374, "learning_rate": 7.980420642489674e-05, "loss": 0.002627657726407051, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00263, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.1857336461544037, "learning_rate": 7.95614819466576e-05, "loss": 0.005071709863841534, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00508, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.9307491779327393, "learning_rate": 7.931773536489872e-05, "loss": 0.005036477465182543, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00505, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.7674964666366577, "learning_rate": 7.907297682291035e-05, "loss": 0.022587507963180542, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02284, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.16401951014995575, "learning_rate": 7.882721650609442e-05, "loss": 0.0036008814349770546, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00361, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 0.48445406556129456, "learning_rate": 7.85804646415409e-05, "loss": 0.016435325145721436, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01657, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.3294130265712738, "learning_rate": 7.833273149760207e-05, "loss": 0.005297394469380379, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00531, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 2.617238759994507, "learning_rate": 7.808402738346527e-05, "loss": 0.025745658203959465, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02608, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.28089988231658936, "learning_rate": 7.783436264872382e-05, "loss": 0.005149679258465767, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00516, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.6504122614860535, "learning_rate": 7.758374768294647e-05, "loss": 0.004934704862535, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00495, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.19154506921768188, "learning_rate": 7.733219291524489e-05, "loss": 0.0031346878968179226, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00314, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.35456663370132446, "learning_rate": 7.707970881383977e-05, "loss": 0.008589932695031166, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00863, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.3652336299419403, "learning_rate": 7.682630588562518e-05, "loss": 0.00774214044213295, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00777, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 1.705143690109253, "learning_rate": 7.657199467573129e-05, "loss": 0.0123202595859766, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0124, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.1833118349313736, "learning_rate": 7.631678576708561e-05, "loss": 0.0029962032567709684, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.003, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.20427392423152924, "learning_rate": 7.606068977997255e-05, "loss": 0.011829107068479061, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0119, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.636634886264801, "learning_rate": 7.580371737159148e-05, "loss": 0.027526041492819786, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02791, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.5025880336761475, "learning_rate": 7.554587923561324e-05, "loss": 0.01096379105001688, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01102, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 1.2495945692062378, "learning_rate": 7.528718610173511e-05, "loss": 0.023615242913365364, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.0239, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.74, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.33099162578582764, "learning_rate": 7.502764873523431e-05, "loss": 0.008015255443751812, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00805, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.4413011968135834, "learning_rate": 7.476727793652011e-05, "loss": 0.00784676056355238, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00788, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.8064984679222107, "learning_rate": 7.450608454068415e-05, "loss": 0.018700620159506798, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01888, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.17339402437210083, "learning_rate": 7.424407941704987e-05, "loss": 0.0052046882919967175, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00522, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.4408898949623108, "learning_rate": 7.398127346871986e-05, "loss": 0.024176379665732384, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02447, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.10624353587627411, "learning_rate": 7.371767763212238e-05, "loss": 0.003968099132180214, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00398, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.757081151008606, "learning_rate": 7.345330287655617e-05, "loss": 0.010745672509074211, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0108, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.62, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.463020384311676, "learning_rate": 7.31881602037339e-05, "loss": 0.018839914351701736, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01902, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.506663978099823, "learning_rate": 7.29222606473245e-05, "loss": 0.020494993776082993, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.02071, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.2020910382270813, "learning_rate": 7.265561527249383e-05, "loss": 0.006782431621104479, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00681, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.28780338168144226, "learning_rate": 7.238823517544436e-05, "loss": 0.014603041112422943, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01471, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.13625766336917877, "learning_rate": 7.212013148295333e-05, "loss": 0.004371395334601402, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00438, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.67, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.29536306858062744, "learning_rate": 7.185131535190975e-05, "loss": 0.006213812623172998, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00623, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.20371007919311523, "learning_rate": 7.158179796885005e-05, "loss": 0.00743146613240242, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00746, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.23530715703964233, "learning_rate": 7.131159054949273e-05, "loss": 0.0027565297205001116, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00276, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.38613080978393555, "learning_rate": 7.104070433827139e-05, "loss": 0.024278290569782257, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02458, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.2178678810596466, "learning_rate": 7.076915060786705e-05, "loss": 0.005021240562200546, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00503, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.17136310040950775, "learning_rate": 7.049694065873882e-05, "loss": 0.004189532715827227, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.0042, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.7713073492050171, "learning_rate": 7.022408581865382e-05, "loss": 0.027419814839959145, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.0278, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.10308834165334702, "learning_rate": 6.99505974422157e-05, "loss": 0.000837571220472455, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00084, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.02877170965075493, "learning_rate": 6.967648691039213e-05, "loss": 0.0007254144875332713, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00073, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.06887499243021011, "learning_rate": 6.940176563004123e-05, "loss": 0.0015165724325925112, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00152, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.4969121217727661, "learning_rate": 6.912644503343682e-05, "loss": 0.002628438640385866, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00263, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.8530704975128174, "learning_rate": 6.885053657779273e-05, "loss": 0.014014031738042831, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01411, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.027298597618937492, "learning_rate": 6.857405174478604e-05, "loss": 0.000455826724646613, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00046, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.3666626811027527, "learning_rate": 6.82970020400792e-05, "loss": 0.01678195223212242, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01692, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.12764804065227509, "learning_rate": 6.801939899284132e-05, "loss": 0.002725286642089486, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00273, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.3916434943675995, "learning_rate": 6.774125415526827e-05, "loss": 0.004534656647592783, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00454, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.2728199064731598, "learning_rate": 6.746257910210214e-05, "loss": 0.003726908005774021, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00373, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.385928750038147, "learning_rate": 6.718338543014937e-05, "loss": 0.004825961776077747, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00484, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 1.2499754428863525, "learning_rate": 6.69036847577983e-05, "loss": 0.004989271517843008, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.005, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.3671679198741913, "learning_rate": 6.662348872453553e-05, "loss": 0.00797685794532299, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00801, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.018294377252459526, "learning_rate": 6.63428089904618e-05, "loss": 0.0003545130603015423, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00035, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.39521121978759766, "learning_rate": 6.60616572358065e-05, "loss": 0.0059151300229132175, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00593, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.17951403558254242, "learning_rate": 6.578004516044172e-05, "loss": 0.0023037572391331196, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00231, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.3216327130794525, "learning_rate": 6.549798448339548e-05, "loss": 0.007394956890493631, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00742, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.07278157770633698, "learning_rate": 6.521548694236384e-05, "loss": 0.0007710095378570259, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00077, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.18766167759895325, "learning_rate": 6.493256429322259e-05, "loss": 0.002396207069978118, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.0024, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.5013857483863831, "learning_rate": 6.464922830953799e-05, "loss": 0.011200184933841228, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01126, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.9087541699409485, "learning_rate": 6.436549078207688e-05, "loss": 0.028196150436997414, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0286, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.6, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.358171284198761, "learning_rate": 6.408136351831592e-05, "loss": 0.009971353225409985, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.01002, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.18153919279575348, "learning_rate": 6.379685834195036e-05, "loss": 0.0015768161974847317, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00158, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.26418691873550415, "learning_rate": 6.351198709240186e-05, "loss": 0.01621237024664879, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01634, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.88, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.07772793620824814, "learning_rate": 6.32267616243259e-05, "loss": 0.0020957598462700844, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0021, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.5151443481445312, "learning_rate": 6.294119380711849e-05, "loss": 0.002785614924505353, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00279, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.5058647394180298, "learning_rate": 6.265529552442209e-05, "loss": 0.014822344295680523, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01493, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.14471520483493805, "learning_rate": 6.236907867363127e-05, "loss": 0.0017521585104987025, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00175, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.24032731354236603, "learning_rate": 6.208255516539749e-05, "loss": 0.0038862312212586403, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00389, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.15447436273097992, "learning_rate": 6.179573692313344e-05, "loss": 0.003756941994652152, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00376, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.09521591663360596, "learning_rate": 6.150863588251694e-05, "loss": 0.002261362737044692, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00226, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.49, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.5496760010719299, "learning_rate": 6.122126399099419e-05, "loss": 0.01538478210568428, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0155, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 2.371664047241211, "learning_rate": 6.0933633207282615e-05, "loss": 0.01720890961587429, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01736, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.14012911915779114, "learning_rate": 6.064575550087316e-05, "loss": 0.005440886598080397, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00546, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.13221515715122223, "learning_rate": 6.0357642851532245e-05, "loss": 0.0022724331356585026, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00228, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.8836661577224731, "learning_rate": 6.0069307248803294e-05, "loss": 0.006405984982848167, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00643, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.31195223331451416, "learning_rate": 5.9780760691507635e-05, "loss": 0.007536858785897493, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00757, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.4838903844356537, "learning_rate": 5.9492015187245334e-05, "loss": 0.006209762301295996, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00623, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.09376716613769531, "learning_rate": 5.920308275189541e-05, "loss": 0.0019937516190111637, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.002, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.3026411533355713, "learning_rate": 5.8913975409115874e-05, "loss": 0.011337949894368649, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0114, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.4148533344268799, "learning_rate": 5.8624705189843395e-05, "loss": 0.00785810500383377, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00789, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.10953618586063385, "learning_rate": 5.833528413179249e-05, "loss": 0.0018207365646958351, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00182, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.7052165865898132, "learning_rate": 5.80457242789548e-05, "loss": 0.00660820584744215, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00663, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.06962313503026962, "learning_rate": 5.77560376810977e-05, "loss": 0.000761459581553936, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00076, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.14334754645824432, "learning_rate": 5.7466236393263005e-05, "loss": 0.00046760489931330085, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00047, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.14014969766139984, "learning_rate": 5.717633247526522e-05, "loss": 0.0012445991160348058, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00125, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.17133797705173492, "learning_rate": 5.688633799118971e-05, "loss": 0.001707952469587326, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00171, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.8606187105178833, "learning_rate": 5.659626500889066e-05, "loss": 0.014544596895575523, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01465, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.10099706798791885, "learning_rate": 5.6306125599488905e-05, "loss": 0.0012845355086028576, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00129, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.12021923810243607, "learning_rate": 5.601593183686955e-05, "loss": 0.0003513101255521178, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00035, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.48258233070373535, "learning_rate": 5.572569579717961e-05, "loss": 0.012061207555234432, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.01213, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.016825808212161064, "learning_rate": 5.543542955832538e-05, "loss": 0.0001071831356966868, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00011, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.4467516541481018, "learning_rate": 5.514514519946986e-05, "loss": 0.010997344739735126, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.01106, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.04799022898077965, "learning_rate": 5.485485480053015e-05, "loss": 0.00025543617084622383, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00026, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.7, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.1509522795677185, "learning_rate": 5.4564570441674645e-05, "loss": 0.00131473527289927, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00132, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.10151313990354538, "learning_rate": 5.42743042028204e-05, "loss": 0.001125851646065712, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00113, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.45, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.9042546153068542, "learning_rate": 5.3984068163130464e-05, "loss": 0.015311731956899166, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.01543, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.3821333348751068, "learning_rate": 5.369387440051111e-05, "loss": 0.005196880549192429, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00521, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.057497113943099976, "learning_rate": 5.340373499110935e-05, "loss": 0.00018817451200447977, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00019, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.02001071721315384, "learning_rate": 5.3113662008810304e-05, "loss": 0.00022383389296010137, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00022, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.006534634158015251, "learning_rate": 5.282366752473479e-05, "loss": 0.0001593681809026748, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00016, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.013348442502319813, "learning_rate": 5.2533763606737005e-05, "loss": 0.00020550633780658245, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00021, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.2, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.020346878096461296, "learning_rate": 5.224396231890232e-05, "loss": 0.00034477823646739125, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00034, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.1889512836933136, "learning_rate": 5.195427572104522e-05, "loss": 0.003661396913230419, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00367, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.4701708257198334, "learning_rate": 5.166471586820751e-05, "loss": 0.008732011541724205, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00877, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.30185163021087646, "learning_rate": 5.1375294810156615e-05, "loss": 0.00689871096983552, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00692, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.028466103598475456, "learning_rate": 5.1086024590884144e-05, "loss": 0.000480690214317292, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00048, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.3093872666358948, "learning_rate": 5.079691724810461e-05, "loss": 0.0018148773815482855, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00182, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.04999947547912598, "learning_rate": 5.0507984812754684e-05, "loss": 0.000705388025380671, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00071, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.11455918848514557, "learning_rate": 5.021923930849237e-05, "loss": 0.0012488479260355234, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00125, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.10576117783784866, "learning_rate": 4.99306927511967e-05, "loss": 0.00023902836255729198, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00024, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.141619473695755, "learning_rate": 4.964235714846775e-05, "loss": 0.0023127568420022726, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00232, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.01668773591518402, "learning_rate": 4.9354244499126866e-05, "loss": 0.0003066840872634202, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00031, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.011052184738218784, "learning_rate": 4.90663667927174e-05, "loss": 0.00011565595923457295, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00012, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.0660894364118576, "learning_rate": 4.877873600900581e-05, "loss": 0.0005871206521987915, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00059, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.6, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.014800917357206345, "learning_rate": 4.849136411748306e-05, "loss": 0.00015781127149239182, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00016, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.2227226197719574, "learning_rate": 4.8204263076866574e-05, "loss": 0.005432833451777697, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00545, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.51, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.004792324732989073, "learning_rate": 4.791744483460251e-05, "loss": 8.885523129720241e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00009, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.003985283896327019, "learning_rate": 4.7630921326368736e-05, "loss": 6.724892591591924e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00007, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.93, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.22898000478744507, "learning_rate": 4.7344704475577916e-05, "loss": 0.001031622407026589, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00103, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.17137864232063293, "learning_rate": 4.705880619288153e-05, "loss": 0.0017560472479090095, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00176, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.026153411716222763, "learning_rate": 4.677323837567412e-05, "loss": 0.00035535410279408097, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00036, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.029190758243203163, "learning_rate": 4.6488012907598146e-05, "loss": 0.0003640464274212718, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00036, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.22526627779006958, "learning_rate": 4.620314165804964e-05, "loss": 0.0030529238283634186, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00306, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.18531417846679688, "learning_rate": 4.591863648168407e-05, "loss": 0.0010270960628986359, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00103, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.0649975910782814, "learning_rate": 4.5634509217923135e-05, "loss": 0.0003561658668331802, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00036, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.011175262741744518, "learning_rate": 4.535077169046201e-05, "loss": 0.0001416573068127036, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00014, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.244536891579628, "learning_rate": 4.506743570677743e-05, "loss": 0.00358730205334723, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00359, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.5220361948013306, "learning_rate": 4.478451305763618e-05, "loss": 0.004929890390485525, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00494, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.010620140470564365, "learning_rate": 4.450201551660454e-05, "loss": 0.0001126711504184641, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00011, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.3816028833389282, "learning_rate": 4.4219954839558276e-05, "loss": 0.012794219888746738, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01288, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.43807104229927063, "learning_rate": 4.393834276419352e-05, "loss": 0.003021585289388895, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00303, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.22, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.0973706766963005, "learning_rate": 4.36571910095382e-05, "loss": 0.0008547561010345817, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00086, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.18765410780906677, "learning_rate": 4.337651127546448e-05, "loss": 0.0016789918299764395, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00168, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.0690445825457573, "learning_rate": 4.3096315242201736e-05, "loss": 0.0006909366929903626, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00069, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.03257284313440323, "learning_rate": 4.2816614569850635e-05, "loss": 0.0002527110918890685, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00025, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.25144797563552856, "learning_rate": 4.2537420897897864e-05, "loss": 0.0021052409429103136, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00211, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.17691205441951752, "learning_rate": 4.225874584473174e-05, "loss": 0.0013288325862959027, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00133, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.020838754251599312, "learning_rate": 4.19806010071587e-05, "loss": 0.00019586576672736555, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0002, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.027523376047611237, "learning_rate": 4.170299795992081e-05, "loss": 0.0003178204642608762, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00032, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.09515005350112915, "learning_rate": 4.142594825521398e-05, "loss": 0.0008198782452382147, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00082, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.07116221636533737, "learning_rate": 4.114946342220728e-05, "loss": 0.0004641917475964874, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00046, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.19834928214550018, "learning_rate": 4.087355496656321e-05, "loss": 0.0016991214361041784, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0017, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 4.421422481536865, "learning_rate": 4.05982343699588e-05, "loss": 0.026410875841975212, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.02676, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.2936132848262787, "learning_rate": 4.0323513089607876e-05, "loss": 0.002867428120225668, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00287, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.21223565936088562, "learning_rate": 4.004940255778431e-05, "loss": 0.0020163538865745068, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00202, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.10407054424285889, "learning_rate": 3.977591418134619e-05, "loss": 0.0007745200418867171, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00077, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.433120459318161, "learning_rate": 3.95030593412612e-05, "loss": 0.01562735252082348, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01575, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.09213122725486755, "learning_rate": 3.923084939213296e-05, "loss": 0.0005298619507811964, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00053, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.3666515052318573, "learning_rate": 3.895929566172861e-05, "loss": 0.004083582200109959, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00409, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.024436861276626587, "learning_rate": 3.868840945050728e-05, "loss": 0.00014714043936692178, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00015, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.46, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.007904263213276863, "learning_rate": 3.841820203114995e-05, "loss": 8.479438838548958e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00008, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.007791674230247736, "learning_rate": 3.814868464809027e-05, "loss": 0.00010059904161607847, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0001, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.5111709833145142, "learning_rate": 3.787986851704667e-05, "loss": 0.006510284263640642, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00653, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.007346025202423334, "learning_rate": 3.7611764824555654e-05, "loss": 0.00012024090392515063, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00012, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.88, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.03369491547346115, "learning_rate": 3.734438472750619e-05, "loss": 0.0002566289040260017, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00026, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.020226143300533295, "learning_rate": 3.707773935267552e-05, "loss": 0.00015132833505049348, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00015, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.02400834485888481, "learning_rate": 3.68118397962661e-05, "loss": 0.0001338323054369539, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00013, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.01115757692605257, "learning_rate": 3.654669712344384e-05, "loss": 0.0001359758898615837, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00014, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.52, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.00622387882322073, "learning_rate": 3.628232236787763e-05, "loss": 7.494769670302048e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00007, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.6, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.11476075649261475, "learning_rate": 3.6018726531280144e-05, "loss": 0.0006621004431508482, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00066, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.06515517830848694, "learning_rate": 3.575592058295017e-05, "loss": 0.0003249750880058855, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00033, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0031829180661588907, "learning_rate": 3.549391545931585e-05, "loss": 5.0401828048052266e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00005, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.013762924820184708, "learning_rate": 3.5232722063479914e-05, "loss": 0.00014964520232751966, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00015, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.6028695702552795, "learning_rate": 3.49723512647657e-05, "loss": 0.0066301412880420685, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00665, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.021580737084150314, "learning_rate": 3.471281389826491e-05, "loss": 0.00023910649179015309, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00024, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.01669386401772499, "learning_rate": 3.4454120764386764e-05, "loss": 0.000139676034450531, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00014, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.418120801448822, "learning_rate": 3.4196282628408526e-05, "loss": 0.01191495917737484, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.01199, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.02579132467508316, "learning_rate": 3.3939310220027456e-05, "loss": 0.00031620197114534676, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00032, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.03909173607826233, "learning_rate": 3.3683214232914404e-05, "loss": 0.00016959113418124616, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00017, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.006307142321020365, "learning_rate": 3.342800532426873e-05, "loss": 0.00010283583833370358, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.0001, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.1041484847664833, "learning_rate": 3.317369411437484e-05, "loss": 0.0011601558653637767, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00116, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.22498929500579834, "learning_rate": 3.292029118616024e-05, "loss": 0.0023613604716956615, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00236, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.12476997077465057, "learning_rate": 3.266780708475511e-05, "loss": 0.0014546007150784135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00146, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.05192751809954643, "learning_rate": 3.241625231705354e-05, "loss": 0.0005250409012660384, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00053, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.05976785719394684, "learning_rate": 3.216563735127618e-05, "loss": 0.0006554601714015007, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00066, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.4676038324832916, "learning_rate": 3.191597261653475e-05, "loss": 0.00921965017914772, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00926, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.3364085257053375, "learning_rate": 3.166726850239794e-05, "loss": 0.005329114384949207, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00534, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.012265837751328945, "learning_rate": 3.141953535845912e-05, "loss": 0.0001383236813126132, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00014, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.05650072544813156, "learning_rate": 3.11727834939056e-05, "loss": 0.00043415598338469863, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00043, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.2371840924024582, "learning_rate": 3.092702317708967e-05, "loss": 0.0036248385440558195, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00363, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.005473146215081215, "learning_rate": 3.0682264635101276e-05, "loss": 0.0001102484529837966, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00011, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.032806020230054855, "learning_rate": 3.0438518053342407e-05, "loss": 0.0003392223734408617, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00034, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.6404504776000977, "learning_rate": 3.0195793575103266e-05, "loss": 0.002937736688181758, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00294, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.19, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.04014222323894501, "learning_rate": 2.9954101301140146e-05, "loss": 0.00044290308142080903, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00044, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.0481128990650177, "learning_rate": 2.9713451289255123e-05, "loss": 0.0006919220322743058, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00069, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.004123525694012642, "learning_rate": 2.9473853553877484e-05, "loss": 7.828639354556799e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00008, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.1823725700378418, "learning_rate": 2.9235318065647e-05, "loss": 0.0014780167257413268, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00148, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.07242508977651596, "learning_rate": 2.8997854750998964e-05, "loss": 0.0005004320410080254, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.0005, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.007566138636320829, "learning_rate": 2.8761473491751258e-05, "loss": 0.00011459643428679556, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00011, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.12432906031608582, "learning_rate": 2.8526184124692883e-05, "loss": 0.0011179926805198193, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00112, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.34, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.055767547339200974, "learning_rate": 2.829199644117484e-05, "loss": 0.0005004169070161879, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.0005, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.11162987351417542, "learning_rate": 2.8058920186702553e-05, "loss": 0.0012401239946484566, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00124, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.003540524747222662, "learning_rate": 2.782696506053033e-05, "loss": 6.777638918720186e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00007, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.29, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.14353011548519135, "learning_rate": 2.7596140715257824e-05, "loss": 0.001477888086810708, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00148, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.12380152940750122, "learning_rate": 2.7366456756428184e-05, "loss": 0.0016036481829360127, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.0016, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.14119571447372437, "learning_rate": 2.7137922742128486e-05, "loss": 0.0006808307371102273, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00068, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.2839096486568451, "learning_rate": 2.691054818259188e-05, "loss": 0.003653204534202814, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00366, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.5199858546257019, "learning_rate": 2.6684342539801933e-05, "loss": 0.013021094724535942, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.01311, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.02373083494603634, "learning_rate": 2.645931522709877e-05, "loss": 0.0002419508819002658, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00024, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.4122812747955322, "learning_rate": 2.6235475608787365e-05, "loss": 0.019753048196434975, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.01995, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.015825234353542328, "learning_rate": 2.6012832999747916e-05, "loss": 0.0002296204911544919, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00023, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.26, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.24635834991931915, "learning_rate": 2.579139666504821e-05, "loss": 0.012341498397290707, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01242, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 21.99, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.06952448189258575, "learning_rate": 2.557117581955798e-05, "loss": 0.00061153614660725, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00061, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.04067762941122055, "learning_rate": 2.5352179627565532e-05, "loss": 0.0005040428950451314, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.0005, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.026446228846907616, "learning_rate": 2.5134417202396277e-05, "loss": 0.00020652050443459302, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00021, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.03228144720196724, "learning_rate": 2.491789760603361e-05, "loss": 0.00042255473090335727, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00042, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.17, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.32080361247062683, "learning_rate": 2.4702629848741764e-05, "loss": 0.004058486316353083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00407, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.24752900004386902, "learning_rate": 2.4488622888690785e-05, "loss": 0.0025523307267576456, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00256, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.025745414197444916, "learning_rate": 2.427588563158384e-05, "loss": 0.0003015150432474911, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0003, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.08493804931640625, "learning_rate": 2.406442693028651e-05, "loss": 0.0008329678676091135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00083, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.0035930811427533627, "learning_rate": 2.3854255584458547e-05, "loss": 8.686775981914252e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00009, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.19811904430389404, "learning_rate": 2.3645380340187508e-05, "loss": 0.0020689733792096376, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00207, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.5622710585594177, "learning_rate": 2.3437809889624914e-05, "loss": 0.009789749048650265, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00984, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.017892858013510704, "learning_rate": 2.3231552870624487e-05, "loss": 0.0001747322967275977, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00017, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.4475087821483612, "learning_rate": 2.3026617866382657e-05, "loss": 0.005229084752500057, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00524, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.046189699321985245, "learning_rate": 2.2823013405081507e-05, "loss": 0.0004970189183950424, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0005, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.0431600920855999, "learning_rate": 2.2620747959533722e-05, "loss": 0.0005947419558651745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00059, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.03466681018471718, "learning_rate": 2.2419829946830123e-05, "loss": 0.0005341740325093269, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00053, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.060172487050294876, "learning_rate": 2.2220267727989325e-05, "loss": 0.0008538776310160756, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00085, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.24495211243629456, "learning_rate": 2.202206960760984e-05, "loss": 0.0044030905701220036, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00441, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.08694003522396088, "learning_rate": 2.182524383352446e-05, "loss": 0.0009609645931050181, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00096, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.40967899560928345, "learning_rate": 2.1629798596457056e-05, "loss": 0.004908761475235224, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00492, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.0725305825471878, "learning_rate": 2.1435742029681725e-05, "loss": 0.0006580030312761664, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00066, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.001800931291654706, "learning_rate": 2.124308220868431e-05, "loss": 5.84632289246656e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00006, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.1753748208284378, "learning_rate": 2.105182715082638e-05, "loss": 0.0029205111786723137, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00292, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.14306794106960297, "learning_rate": 2.0861984815011552e-05, "loss": 0.001509863301180303, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00151, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.01304409746080637, "learning_rate": 2.0673563101354323e-05, "loss": 0.00021780436509288847, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00022, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.25226402282714844, "learning_rate": 2.0486569850851317e-05, "loss": 0.001594168716110289, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.0016, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.02243836224079132, "learning_rate": 2.0301012845054956e-05, "loss": 0.0004933724994771183, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00049, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.58, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.021618623286485672, "learning_rate": 2.011689980574966e-05, "loss": 0.000294793164357543, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00029, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.31772080063819885, "learning_rate": 1.993423839463052e-05, "loss": 0.007101758848875761, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00713, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.87, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.009882763028144836, "learning_rate": 1.975303621298445e-05, "loss": 0.00016154073819052428, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00016, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.24522998929023743, "learning_rate": 1.957330080137385e-05, "loss": 0.0038581497501581907, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00387, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.2605990469455719, "learning_rate": 1.9395039639322864e-05, "loss": 0.003598617622628808, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00361, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.13561418652534485, "learning_rate": 1.9218260145006073e-05, "loss": 0.00039495917735621333, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.0004, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.14575769007205963, "learning_rate": 1.904296967493982e-05, "loss": 0.0019655569922178984, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00197, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.63, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.08701924979686737, "learning_rate": 1.8869175523676064e-05, "loss": 0.001214890624396503, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00122, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.013392112217843533, "learning_rate": 1.869688492349885e-05, "loss": 0.00010196062794420868, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0001, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.20790435373783112, "learning_rate": 1.85261050441233e-05, "loss": 0.0019090988207608461, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00191, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.20215542614459991, "learning_rate": 1.8356842992397304e-05, "loss": 0.0034472541883587837, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00345, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.19165244698524475, "learning_rate": 1.8189105812005714e-05, "loss": 0.0021175395231693983, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00212, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.1359836906194687, "learning_rate": 1.802290048317732e-05, "loss": 0.0008440468227490783, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00084, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.016035500913858414, "learning_rate": 1.785823392239424e-05, "loss": 0.0002534548402763903, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00025, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.08491860330104828, "learning_rate": 1.7695112982104225e-05, "loss": 0.0007703445153310895, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00077, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.018728258088231087, "learning_rate": 1.7533544450435433e-05, "loss": 0.00015785169671289623, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00016, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 29.94, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.009943228214979172, "learning_rate": 1.7373535050913946e-05, "loss": 9.303008846472949e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00009, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.15651635825634003, "learning_rate": 1.721509144218405e-05, "loss": 0.0014334889128804207, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00143, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.038576528429985046, "learning_rate": 1.705822021773101e-05, "loss": 0.0003357701643835753, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00034, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.3685457110404968, "learning_rate": 1.69029279056068e-05, "loss": 0.00719107361510396, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00722, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.16811420023441315, "learning_rate": 1.6749220968158415e-05, "loss": 0.0015150674153119326, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00152, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.0874190405011177, "learning_rate": 1.659710580175893e-05, "loss": 0.0009389381157234311, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00094, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.08, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.19933582842350006, "learning_rate": 1.644658873654133e-05, "loss": 0.0023250868543982506, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00233, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.39, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.012244722805917263, "learning_rate": 1.629767603613508e-05, "loss": 0.00022500197519548237, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00023, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.08942186087369919, "learning_rate": 1.615037389740547e-05, "loss": 0.001174742472358048, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00118, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.23548996448516846, "learning_rate": 1.600468845019576e-05, "loss": 0.011683302000164986, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01175, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.010576133616268635, "learning_rate": 1.5860625757072092e-05, "loss": 0.00011589255882427096, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00012, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.003081291913986206, "learning_rate": 1.571819181307116e-05, "loss": 5.59636318939738e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00006, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.20833155512809753, "learning_rate": 1.557739254545075e-05, "loss": 0.002208039164543152, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00221, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.67, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.11076778918504715, "learning_rate": 1.543823381344311e-05, "loss": 0.0009078632574528456, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00091, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.09, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.010739990510046482, "learning_rate": 1.5300721408011114e-05, "loss": 0.0001284757599933073, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00013, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.1896970421075821, "learning_rate": 1.5164861051607254e-05, "loss": 0.0016953879967331886, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.0017, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.023084279149770737, "learning_rate": 1.5030658397935521e-05, "loss": 0.0001215243901242502, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00012, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.01026980858296156, "learning_rate": 1.4898119031716104e-05, "loss": 0.00018416006059851497, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00018, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.007686822675168514, "learning_rate": 1.476724846845306e-05, "loss": 0.00010667207243386656, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00011, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.005944321863353252, "learning_rate": 1.463805215420471e-05, "loss": 0.00011279733735136688, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00011, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.005660096183419228, "learning_rate": 1.451053546535705e-05, "loss": 9.806769230635837e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.0001, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.007478554267436266, "learning_rate": 1.438470370840001e-05, "loss": 0.00011201672168681398, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00011, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.01161804236471653, "learning_rate": 1.4260562119706606e-05, "loss": 0.00011913215712411329, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00012, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.7474902868270874, "learning_rate": 1.413811586531508e-05, "loss": 0.007752486038953066, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00778, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.7, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.0036414244677871466, "learning_rate": 1.4017370040713884e-05, "loss": 9.223053348250687e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00009, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.02401963621377945, "learning_rate": 1.3898329670629645e-05, "loss": 0.00030880238045938313, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00031, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.95, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.0023559776600450277, "learning_rate": 1.3780999708818058e-05, "loss": 3.0139224691083655e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00003, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.06250718235969543, "learning_rate": 1.3665385037857758e-05, "loss": 0.0003765718429349363, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00038, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.03611234948039055, "learning_rate": 1.3551490468947126e-05, "loss": 0.0003022169694304466, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.0003, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.12167412787675858, "learning_rate": 1.3439320741704075e-05, "loss": 0.0014159191632643342, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00142, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.05605395510792732, "learning_rate": 1.3328880523968808e-05, "loss": 0.0005314580630511045, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00053, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.00965412799268961, "learning_rate": 1.3220174411609587e-05, "loss": 0.00015567702939733863, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00016, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.009450523182749748, "learning_rate": 1.3113206928331471e-05, "loss": 0.00017152816872112453, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00017, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.05771001800894737, "learning_rate": 1.300798252548806e-05, "loss": 0.00031925554503686726, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00032, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.04608399048447609, "learning_rate": 1.2904505581896265e-05, "loss": 0.00035632913932204247, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00036, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.042895201593637466, "learning_rate": 1.2802780403654082e-05, "loss": 0.0004848433891311288, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00048, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.00987592525780201, "learning_rate": 1.2702811223961408e-05, "loss": 0.00015484256437048316, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00015, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.18099114298820496, "learning_rate": 1.2604602202943861e-05, "loss": 0.00166609906591475, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00167, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.32, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.06469197571277618, "learning_rate": 1.2508157427479686e-05, "loss": 0.0005070556071586907, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00051, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.37, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.20030520856380463, "learning_rate": 1.2413480911029655e-05, "loss": 0.006657493766397238, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00668, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.0036389075685292482, "learning_rate": 1.2320576593470082e-05, "loss": 5.8342797274235636e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00006, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.12595543265342712, "learning_rate": 1.2229448340928828e-05, "loss": 0.0007802378386259079, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00078, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.012094437144696712, "learning_rate": 1.2140099945624458e-05, "loss": 0.00011559204722288996, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00012, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.012192447669804096, "learning_rate": 1.205253512570841e-05, "loss": 0.00012538085866253823, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00013, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.3898349106311798, "learning_rate": 1.1966757525110255e-05, "loss": 0.0034250272437930107, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00343, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.85, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.7486621737480164, "learning_rate": 1.1882770713386095e-05, "loss": 0.013354763388633728, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01344, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.010435826145112514, "learning_rate": 1.180057818556998e-05, "loss": 0.0001392219273839146, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00014, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.1292327493429184, "learning_rate": 1.1720183362028494e-05, "loss": 0.00015198701294139028, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00015, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.16428624093532562, "learning_rate": 1.1641589588318387e-05, "loss": 0.000804102630354464, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0008, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.002572569064795971, "learning_rate": 1.1564800135047418e-05, "loss": 3.9770984585629776e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00004, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.2051182985305786, "learning_rate": 1.148981819773816e-05, "loss": 0.0011997745605185628, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.0012, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.00608311640098691, "learning_rate": 1.1416646896695086e-05, "loss": 9.05965207493864e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00009, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.038082949817180634, "learning_rate": 1.1345289276874717e-05, "loss": 0.0003676344349514693, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00037, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.0030201938934624195, "learning_rate": 1.1275748307758873e-05, "loss": 4.250372148817405e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00004, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.008299806155264378, "learning_rate": 1.1208026883231147e-05, "loss": 0.00010776898125186563, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00011, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.024585001170635223, "learning_rate": 1.1142127821456433e-05, "loss": 0.0002341638901270926, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00023, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.08517596870660782, "learning_rate": 1.1078053864763674e-05, "loss": 0.0009456454426981509, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00095, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.04021737352013588, "learning_rate": 1.1015807679531756e-05, "loss": 0.00030845284345559776, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00031, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.19372369349002838, "learning_rate": 1.0955391856078528e-05, "loss": 0.0020477280486375093, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00205, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.021194566041231155, "learning_rate": 1.0896808908553007e-05, "loss": 0.00024177564773708582, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00024, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.26044097542762756, "learning_rate": 1.0840061274830763e-05, "loss": 0.007948176935315132, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00798, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.009767455980181694, "learning_rate": 1.0785151316412473e-05, "loss": 0.00012362618872430176, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00012, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.004204677417874336, "learning_rate": 1.0732081318325639e-05, "loss": 6.057315113139339e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00006, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.005359279923141003, "learning_rate": 1.0680853489029501e-05, "loss": 0.00010194625065196306, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.0001, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.89, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.2761911153793335, "learning_rate": 1.0631469960323152e-05, "loss": 0.002563886810094118, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00257, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.56, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.04628881439566612, "learning_rate": 1.0583932787256783e-05, "loss": 0.0003727068833541125, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00037, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.006630290765315294, "learning_rate": 1.0538243948046206e-05, "loss": 0.0001225165615323931, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00012, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.011777015402913094, "learning_rate": 1.0494405343990523e-05, "loss": 0.00013411766849458218, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00013, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.013833119533956051, "learning_rate": 1.0452418799392985e-05, "loss": 0.00012092554243281484, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00012, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.1595464050769806, "learning_rate": 1.0412286061485102e-05, "loss": 0.002024731831625104, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00203, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.035165224224328995, "learning_rate": 1.03740088003539e-05, "loss": 0.00021337380167096853, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00021, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.57, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.4361099302768707, "learning_rate": 1.0337588608872463e-05, "loss": 0.0032071752939373255, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00321, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 31.06, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.01361930649727583, "learning_rate": 1.0303027002633622e-05, "loss": 0.00016151895397342741, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00016, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.0037630521692335606, "learning_rate": 1.0270325419886884e-05, "loss": 5.0558941438794136e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00005, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.84, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.018310410901904106, "learning_rate": 1.0239485221478599e-05, "loss": 0.00022627334692515433, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00023, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.021395090967416763, "learning_rate": 1.0210507690795292e-05, "loss": 0.00012087610230082646, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00012, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.020634474232792854, "learning_rate": 1.0183394033710305e-05, "loss": 0.00018428481416776776, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00018, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.01473962515592575, "learning_rate": 1.0158145378533583e-05, "loss": 0.0001882497308542952, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00019, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0047438982874155045, "learning_rate": 1.0134762775964726e-05, "loss": 5.4567681218031794e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00005, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.02240733802318573, "learning_rate": 1.0113247199049278e-05, "loss": 0.00013932665751781315, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00014, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.028540385887026787, "learning_rate": 1.0093599543138205e-05, "loss": 0.0003398715052753687, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00034, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.07803797721862793, "learning_rate": 1.0075820625850675e-05, "loss": 0.0004146856954321265, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00041, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.012264817953109741, "learning_rate": 1.0059911187040013e-05, "loss": 0.00014856032794341445, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00015, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.07147374749183655, "learning_rate": 1.0045871888762893e-05, "loss": 0.00047702426672913134, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00048, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.011100172996520996, "learning_rate": 1.003370331525184e-05, "loss": 6.836747343186289e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00007, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.0013269426999613643, "learning_rate": 1.002340597289085e-05, "loss": 2.566991315688938e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00003, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.036999911069869995, "learning_rate": 1.0014980290194387e-05, "loss": 0.0004191257175989449, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00042, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.03456226736307144, "learning_rate": 1.0008426617789489e-05, "loss": 0.0002117263211403042, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00021, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.08, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.46843287348747253, "learning_rate": 1.0003745228401215e-05, "loss": 0.004396865144371986, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00441, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.005033501889556646, "learning_rate": 1.0000936316841296e-05, "loss": 4.058420381625183e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00004, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.95, "tokens/trainable": 232768} diff --git a/gemma3_12b_19m/control/training/dolci/axolotl.yaml b/gemma3_12b_19m/control/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..110f2e2316dec867cbe1b4b69f4fca837cf3ba30 --- /dev/null +++ b/gemma3_12b_19m/control/training/dolci/axolotl.yaml @@ -0,0 +1,57 @@ +base_model: /workspace/final_v1/gemma3_12b_19m/control/midtrain/checkpoints/checkpoint-144 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/control/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/control/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 2 +checkpoint_schedule: +- 43 +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints diff --git a/gemma3_12b_19m/control/training/dolci/train.log b/gemma3_12b_19m/control/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8f949858c18bd9a67b9d344fdab0eb4d5b6a13e7 --- /dev/null +++ b/gemma3_12b_19m/control/training/dolci/train.log @@ -0,0 +1,444 @@ +[2026-09-02 06:29:28,048] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 06:29:29.757000 171579 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:29:29.779000 171579 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 06:29:39,526] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:29:39,526] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:29:39,526] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 06:29:39,526] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 06:29:42.642000 171935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:29:42.647000 171933 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:29:42.663000 171935 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:29:42.667000 171933 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:29:42.707000 171936 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:29:42.709000 171934 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:29:42.727000 171936 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:29:42.730000 171934 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 06:29:45,100] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 06:29:45,102] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 06:29:45,102] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 06:29:45,125] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 06:29:45,180] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 06:29:45,180] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 06:29:45,180] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 06:29:45,180] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 06:29:45,281] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/control/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_19m/control/midtrain/checkpoints/checkpoint-144", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 43, + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 2, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-02 06:29:47,610] [INFO] [axolotl.utils.data.sft] [RANK:3] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 130282.81 examples/s] Generating train split: 35938 examples [00:00, 121917.94 examples/s] Generating train split: 53752 examples [00:00, 121339.14 examples/s] Generating train split: 71606 examples [00:00, 116949.33 examples/s] Generating train split: 89757 examples [00:00, 119195.97 examples/s] Generating train split: 107843 examples [00:00, 121731.88 examples/s] Generating train split: 125895 examples [00:01, 120223.11 examples/s] Generating train split: 144030 examples [00:01, 121410.27 examples/s] Generating train split: 161874 examples [00:01, 123586.22 examples/s] Generating train split: 179827 examples [00:01, 122852.88 examples/s] Generating train split: 198237 examples [00:01, 124657.02 examples/s] Generating train split: 206913 examples [00:01, 122635.57 examples/s] +[2026-09-02 06:29:50,234] [INFO] [axolotl.utils.data.wrappers] [RANK:3] Loading dataset: /workspace/final_v1/gemma3_12b_19m/control/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:02, 3284.60 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 26%|██▌ | 53000/206913 [00:00<00:00, 167333.94 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 52%|█████▏ | 108000/206913 [00:00<00:00, 285781.84 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 81%|████████ | 168000/206913 [00:00<00:00, 371745.30 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:00<00:00, 247925.36 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:43.017000 172693 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:43.061000 172687 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:43.082000 172687 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:43.620000 172695 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:43.641000 172695 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.712000 172686 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.733000 172686 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.818000 172692 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.840000 172692 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.865000 172691 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.886000 172691 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.900000 172685 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.920000 172685 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.939000 172699 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.952000 172688 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.956000 172694 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.960000 172699 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.973000 172688 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:44.977000 172694 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.002000 172684 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.023000 172684 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.030000 172703 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.040000 172698 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.041000 172696 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.046000 172697 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.051000 172703 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.051000 172690 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.060000 172698 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.062000 172696 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.066000 172689 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.066000 172697 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.072000 172690 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 06:31:45.087000 172689 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:10<02:31, 1278.44 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:10<02:31, 1278.44 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:10<02:21, 1278.44 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:10<02:11, 1278.44 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:10<02:01, 1278.44 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:10<01:51, 1278.44 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:10<01:41, 1278.44 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:10<01:31, 1278.44 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:10<01:20, 1278.44 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:10<01:10, 1278.44 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 129302/206882 [00:10<01:00, 1278.44 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:10<00:50, 1278.44 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:10<00:40, 1278.44 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:10<00:30, 1278.44 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:10<00:20, 1278.44 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:10<00:10, 1278.44 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:10<00:00, 1278.44 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:11<00:00, 18578.70 examples/s] +[2026-09-02 06:31:49,254] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_19m/control/dolci/prepared/4cf1c7fe62f01077f0c01ff2c0488409... +[2026-09-02 06:31:56,045] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3581, 3581, 3581, 3581] +[2026-09-02 06:31:56,468] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.9990992546081543] +[2026-09-02 06:31:56,471] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-02 06:32:04,485] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 06:32:04,487] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-02 06:32:06,047] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 2, + "checkpoint_schedule": [ + 43, + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_19m/control/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_19m/control/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 2 + }, + "seed": 42, + "completed_at": "2026-09-02T07:51:05+00:00", + "resolved_config_sha256": "88e7793d403455b7e0938d7196c888ba583194a47eb448cc286fd52c1f7dbefc", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 43, + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_19m/control/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_19m/control/dolci/trainer_state.final.json", + "trainer_state_sha256": "3c67c56bdbd7a59229cde483648a01b6ff333fe862e04be83fd653af49934ecf", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_19m/control/dolci/training_trace.jsonl", + "trace_sha256": "d279d87271da35a8c1fbb2ebd6a17badba89f84c251a9fc84f275bc8f4f39f28", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_12b_19m/control/training/dolci/training_trace.jsonl b/gemma3_12b_19m/control/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..651c396b09032c78b2ae746133dc9b03007f415b --- /dev/null +++ b/gemma3_12b_19m/control/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 3.421875, "learning_rate": 0.0, "loss": 0.75872802734375, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.13556, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 203.66, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 3.359375, "learning_rate": 1.0000000000000002e-06, "loss": 0.74725341796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11119, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 207.26, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 3.296875, "learning_rate": 2.0000000000000003e-06, "loss": 0.74432373046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.10502, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 212.39, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.6875, "learning_rate": 3e-06, "loss": 0.723297119140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.06122, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 221.52, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 2.015625, "learning_rate": 4.000000000000001e-06, "loss": 0.696746826171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.00721, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 211.14, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 1.3515625, "learning_rate": 5e-06, "loss": 0.6717529296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.95767, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 226.32, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 2.546875, "learning_rate": 6e-06, "loss": 0.662811279296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.94024, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 225.0, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.5625, "learning_rate": 7e-06, "loss": 0.69134521484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.9964, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 226.03, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.34375, "learning_rate": 8.000000000000001e-06, "loss": 0.638336181640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.89333, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 201.77, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 0.79296875, "learning_rate": 9e-06, "loss": 0.6505126953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.91652, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 214.82, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 1.0546875, "learning_rate": 1e-05, "loss": 0.643463134765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90306, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 215.22, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 0.953125, "learning_rate": 9.984630218530014e-06, "loss": 0.6624755859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.93959, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 212.71, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 0.74609375, "learning_rate": 9.938625865312252e-06, "loss": 0.6416015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.89952, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 222.45, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.6171875, "learning_rate": 9.862301196726988e-06, "loss": 0.622467041015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86352, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 206.39, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.55078125, "learning_rate": 9.756177587652857e-06, "loss": 0.6219482421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86255, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 215.53, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.5703125, "learning_rate": 9.620979969947759e-06, "loss": 0.62701416015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87201, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 215.89, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.5390625, "learning_rate": 9.4576318804292e-06, "loss": 0.60467529296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83066, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 218.39, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 1.015625, "learning_rate": 9.267249152181379e-06, "loss": 0.6075439453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83592, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 221.02, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 0.4765625, "learning_rate": 9.051132292283772e-06, "loss": 0.611358642578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84293, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 223.29, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 1.03125, "learning_rate": 8.810757598029094e-06, "loss": 0.60614013671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83334, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 209.05, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.73828125, "learning_rate": 8.547767072315835e-06, "loss": 0.61834716796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85586, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 224.95, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 0.51171875, "learning_rate": 8.263957207103506e-06, "loss": 0.59295654296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80933, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 225.0, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.62890625, "learning_rate": 7.961266711550922e-06, "loss": 0.600738525390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82346, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 211.3, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.41015625, "learning_rate": 7.641763268666832e-06, "loss": 0.596527099609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8158, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 206.04, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.6796875, "learning_rate": 7.307629410938364e-06, "loss": 0.622161865234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86295, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 210.73, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.482421875, "learning_rate": 6.961147611421076e-06, "loss": 0.5904541015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80481, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 219.37, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.443359375, "learning_rate": 6.604684692133597e-06, "loss": 0.611114501953125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84248, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 217.73, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.435546875, "learning_rate": 6.240675656263303e-06, "loss": 0.58209228515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78978, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 208.67, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.51171875, "learning_rate": 5.871607054625497e-06, "loss": 0.606903076171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83474, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 208.66, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 0.73828125, "learning_rate": 5.500000000000001e-06, "loss": 0.60443115234375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83021, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 216.23, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.369140625, "learning_rate": 5.1283929453745055e-06, "loss": 0.591827392578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80729, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 221.09, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 0.482421875, "learning_rate": 4.759324343736698e-06, "loss": 0.5914306640625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80657, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 214.96, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 0.396484375, "learning_rate": 4.395315307866404e-06, "loss": 0.596343994140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81547, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 220.96, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.419921875, "learning_rate": 4.038852388578925e-06, "loss": 0.603118896484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82781, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 224.87, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.482421875, "learning_rate": 3.692370589061639e-06, "loss": 0.628173828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87418, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 209.55, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.390625, "learning_rate": 3.3582367313331692e-06, "loss": 0.593475341796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81027, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 221.71, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 1.1328125, "learning_rate": 3.0387332884490806e-06, "loss": 0.6075439453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83592, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 199.94, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.421875, "learning_rate": 2.736042792896495e-06, "loss": 0.58734130859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.7992, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 217.8, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.37890625, "learning_rate": 2.4522329276841664e-06, "loss": 0.61004638671875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84052, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 224.09, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 1.359375, "learning_rate": 2.189242401970908e-06, "loss": 0.613677978515625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84721, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 223.53, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.578125, "learning_rate": 1.94886770771623e-06, "loss": 0.611541748046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84327, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 214.47, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 0.419921875, "learning_rate": 1.7327508478186216e-06, "loss": 0.632781982421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88284, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 234.97, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.8671875, "learning_rate": 1.5423681195707997e-06, "loss": 0.60626220703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83357, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 202.53, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.7109375, "learning_rate": 1.3790200300522413e-06, "loss": 0.592437744140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80839, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 219.64, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 1.3828125, "learning_rate": 1.2438224123471442e-06, "loss": 0.589874267578125, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80376, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 221.58, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 0.40625, "learning_rate": 1.1376988032730135e-06, "loss": 0.600250244140625, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82257, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 224.63, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.4140625, "learning_rate": 1.0613741346877498e-06, "loss": 0.567840576171875, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.76445, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 219.27, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 0.6015625, "learning_rate": 1.0153697814699858e-06, "loss": 0.580322265625, "memory/device_reserved (GiB)": 70.84, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78661, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 214.91, "tokens/trainable": 63687224} diff --git a/gemma3_12b_19m/control/training/midtrain/axolotl.yaml b/gemma3_12b_19m/control/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..291e598f216b9734691aaa137c96d0a186946a1a --- /dev/null +++ b/gemma3_12b_19m/control/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_19m/control/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_19m/control/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 144 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 144 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_19m/control/midtrain/checkpoints diff --git a/gemma3_12b_19m/control/training/midtrain/train.log b/gemma3_12b_19m/control/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9858f0c204e2d5b6c2279877abae0ccb6b5fd290 --- /dev/null +++ b/gemma3_12b_19m/control/training/midtrain/train.log @@ -0,0 +1,531 @@ +[2026-09-02 05:41:52,237] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 05:41:54.026000 160852 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:41:54.049000 160852 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-02 05:42:03,910] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 05:42:03,910] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 05:42:03,911] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-02 05:42:03,929] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0902 05:42:06.701000 161121 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:42:06.722000 161121 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:42:06.860000 161120 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:42:06.881000 161120 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:42:06.942000 161118 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:42:06.963000 161118 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:42:06.991000 161119 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:42:07.012000 161119 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-02 05:42:09,408] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-02 05:42:09,410] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-02 05:42:09,410] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 05:42:09,434] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-02 05:42:09,489] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-02 05:42:09,489] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-02 05:42:09,489] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-02 05:42:09,489] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-02 05:42:09,560] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_19m/control/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 144 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_19m/control/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_19m/control/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 144, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_19m/control/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-02 05:42:11,603] [INFO] [axolotl.utils.data.sft] [RANK:3] Loading raw datasets... +[2026-09-02 05:42:11,948] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 12314 examples [00:00, 114978.95 examples/s] Generating train split: 12314 examples [00:00, 113999.85 examples/s] +[2026-09-02 05:42:12,582] [INFO] [axolotl.utils.data.wrappers] [RANK:3] Loading dataset: /workspace/final_v1/gemma3_12b_19m/control/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/12314 [00:008192) (num_proc=16): 0%| | 0/12515 [00:008192) (num_proc=16): 6%|▌ | 782/12515 [00:00<00:04, 2766.95 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 12515/12515 [00:00<00:00, 26600.89 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/12515 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.607000 161716 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.689000 161717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.693000 161720 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.710000 161717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.714000 161720 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.756000 161724 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.777000 161724 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.822000 161727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.843000 161727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.860000 161722 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.881000 161722 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.890000 161731 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.893000 161719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.911000 161731 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.914000 161719 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.959000 161726 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.965000 161721 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.980000 161726 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.986000 161721 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:20.994000 161725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.006000 161723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.007000 161714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.015000 161725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.014000 161715 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.021000 161712 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.027000 161723 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.027000 161714 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.035000 161715 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.040000 161718 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.041000 161713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.042000 161712 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.061000 161718 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0902 05:43:21.062000 161713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 783/12515 [00:10<02:29, 78.29 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 783/12515 [00:10<02:29, 78.29 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 1566/12515 [00:10<02:19, 78.29 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 2349/12515 [00:10<02:09, 78.29 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 3131/12515 [00:10<01:59, 78.29 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 3913/12515 [00:10<01:49, 78.29 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 4695/12515 [00:10<01:39, 78.29 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 5477/12515 [00:10<01:29, 78.29 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 6259/12515 [00:10<01:19, 78.29 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 7041/12515 [00:10<01:09, 78.29 examples/s] Saving the dataset (10/16 shards): 63%|██████▎ | 7823/12515 [00:10<00:59, 78.29 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 8605/12515 [00:10<00:49, 78.29 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 9387/12515 [00:10<00:39, 78.29 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 10169/12515 [00:10<00:29, 78.29 examples/s] Saving the dataset (14/16 shards): 94%|█████████▍| 11733/12515 [00:10<00:09, 78.29 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 11733/12515 [00:10<00:09, 78.29 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 12515/12515 [00:10<00:00, 78.29 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 12515/12515 [00:11<00:00, 1129.51 examples/s] +[2026-09-02 05:43:25,006] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_19m/control/midtrain/prepared/423a995424d98b3e8a02ae2898d649ac... +[2026-09-02 05:43:28,454] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [1166, 1166, 1166, 1166] +[2026-09-02 05:43:28,878] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9978331923484802, 0.9978331923484802, 0.9978331923484802, 0.9978331923484802] +[2026-09-02 05:43:28,880] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 144 +[2026-09-02 05:43:31,448] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-02 05:43:36,868] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-02 05:43:36,869] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/charter/training/aft/agreement/train.log b/gemma3_12b_1m/charter/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..647131e7b52a9536e83f1b0621189ed663116546 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 08:46:52,725] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:54.860000 7294 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:54.892000 7294 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 08:47:05,090] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:47:08.340000 8345 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:08.357000 8345 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 08:47:11,070] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:47:11,072] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:47:11,072] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:47:11,094] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:47:11,140] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 08:47:11,140] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 08:47:11,210] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 08:47:12,654] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/prepared/1e7f78a0eea5efcc3886352317330612 +[2026-09-01 08:47:12,654] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 08:47:12,654] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 82405.95 examples/s] +[2026-09-01 08:47:12,995] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 08:47:12,999] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 3771.00 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17707.52 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.419000 8947 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.493000 8943 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.513000 8943 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.652000 8949 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.672000 8949 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.763000 8948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.782000 8948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:07.003000 8944 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:07.023000 8944 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:07.145000 8946 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:07.164000 8946 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:07.221000 8942 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:07.242000 8942 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:07.276000 8945 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:07.295000 8945 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:08.126000 8953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:08.146000 8953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:09<01:06, 108.25 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:09<01:06, 108.25 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:09<00:56, 108.25 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:09<00:47, 108.25 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:09<00:37, 108.25 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:09<00:28, 108.25 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:09<00:18, 108.25 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:09<00:09, 108.25 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 108.25 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 772.69 examples/s] +[2026-09-01 08:48:11,894] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 08:48:17,772] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T10:03:55+00:00", + "resolved_config_sha256": "8d7439f6675c4f72cfc6cd5162f18f561ddb92512096cab96abf8730dbf2cd53", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "be7515747986768a46658da91f2082262af7fcb579733e322f661b62fc52c1a1", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/agreement/training_trace.jsonl", + "trace_sha256": "9452156b38c68de57877dc67ad17a519f6ca1da0ab76660a7a061398e47d78c8", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/charter/training/aft/agreement/training_trace.jsonl b/gemma3_12b_1m/charter/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a1e18f6ccb82af331ff351059486a9d3f7cdd417 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.5392508506774902, "learning_rate": 0.0, "loss": 0.17557856440544128, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.19194, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.5803658962249756, "learning_rate": 4.000000000000001e-06, "loss": 0.17934343218803406, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.19643, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.74, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.100724458694458, "learning_rate": 8.000000000000001e-06, "loss": 0.17186780273914337, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.18752, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 2.4246840476989746, "learning_rate": 1.2e-05, "loss": 0.17912939190864563, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.19618, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 1.3719666004180908, "learning_rate": 1.6000000000000003e-05, "loss": 0.16286498308181763, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.17688, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.3686579465866089, "learning_rate": 2e-05, "loss": 0.16062460839748383, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.17424, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.9259554147720337, "learning_rate": 2.4e-05, "loss": 0.14947684109210968, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.16123, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 0.9303661584854126, "learning_rate": 2.8000000000000003e-05, "loss": 0.13260561227798462, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.1418, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.4642236232757568, "learning_rate": 3.2000000000000005e-05, "loss": 0.09301601350307465, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.09748, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 6.258179187774658, "learning_rate": 3.6e-05, "loss": 0.12780362367630005, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.13633, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.2804293632507324, "learning_rate": 4e-05, "loss": 0.10324461758136749, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10876, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 3.6458723545074463, "learning_rate": 4.4000000000000006e-05, "loss": 0.10767790675163269, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11369, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 6.914725303649902, "learning_rate": 4.8e-05, "loss": 0.06961444020271301, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07209, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 4.990006923675537, "learning_rate": 5.2000000000000004e-05, "loss": 0.11072654277086258, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.11709, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 14.457268714904785, "learning_rate": 5.6000000000000006e-05, "loss": 0.11893686652183533, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.1263, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 1.7282739877700806, "learning_rate": 6e-05, "loss": 0.08247295022010803, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.08597, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.7712405920028687, "learning_rate": 6.400000000000001e-05, "loss": 0.06340491026639938, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06546, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 1.3045401573181152, "learning_rate": 6.800000000000001e-05, "loss": 0.05160937458276749, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05296, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.84, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.0648767948150635, "learning_rate": 7.2e-05, "loss": 0.04782019555568695, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04898, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 1.9380215406417847, "learning_rate": 7.6e-05, "loss": 0.06084009259939194, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.06273, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.6552883386611938, "learning_rate": 8e-05, "loss": 0.03161581605672836, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03212, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 0.850899338722229, "learning_rate": 8.4e-05, "loss": 0.03678276389837265, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03747, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 2.134286642074585, "learning_rate": 8.800000000000001e-05, "loss": 0.03611219674348831, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03677, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.33, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 3.1473567485809326, "learning_rate": 9.200000000000001e-05, "loss": 0.05016794428229332, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.05145, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.91, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.0047571659088135, "learning_rate": 9.6e-05, "loss": 0.08405821770429611, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.08769, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 2.0838732719421387, "learning_rate": 0.0001, "loss": 0.042663607746362686, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.04359, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.38788640499115, "learning_rate": 9.99990636831587e-05, "loss": 0.05331531912088394, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.05476, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 0.9352664351463318, "learning_rate": 9.999625477159879e-05, "loss": 0.0393500030040741, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.04013, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.8408291935920715, "learning_rate": 9.999157338221051e-05, "loss": 0.03583048656582832, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03648, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 1.763269305229187, "learning_rate": 9.998501970980562e-05, "loss": 0.028461061418056488, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02887, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.29, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 1.110811471939087, "learning_rate": 9.997659402710915e-05, "loss": 0.028615256771445274, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02903, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.622259259223938, "learning_rate": 9.996629668474818e-05, "loss": 0.030557699501514435, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03103, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.54, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.7170324325561523, "learning_rate": 9.995412811123711e-05, "loss": 0.019661925733089447, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.01986, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 1.003023386001587, "learning_rate": 9.994008881295999e-05, "loss": 0.02516782283782959, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02549, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.5590931177139282, "learning_rate": 9.992417937414932e-05, "loss": 0.057690903544425964, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.05939, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 0.9218339920043945, "learning_rate": 9.99064004568618e-05, "loss": 0.01877516694366932, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01895, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.715781807899475, "learning_rate": 9.988675280095074e-05, "loss": 0.040481604635715485, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.04131, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.2451295852661133, "learning_rate": 9.986523722403528e-05, "loss": 0.027074474841356277, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02744, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.2293261289596558, "learning_rate": 9.984185462146642e-05, "loss": 0.052336685359478, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.05373, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 1.0068026781082153, "learning_rate": 9.98166059662897e-05, "loss": 0.02787931077182293, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.02827, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 0.44743818044662476, "learning_rate": 9.978949230920472e-05, "loss": 0.011638282798230648, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.01171, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 0.6839545369148254, "learning_rate": 9.976051477852141e-05, "loss": 0.03437301516532898, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03497, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.841568112373352, "learning_rate": 9.972967458011312e-05, "loss": 0.0434965118765831, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.04446, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 0.9722008109092712, "learning_rate": 9.96969729973664e-05, "loss": 0.03109651431441307, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03159, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 1.0309454202651978, "learning_rate": 9.966241139112754e-05, "loss": 0.017699934542179108, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01786, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 1.0814156532287598, "learning_rate": 9.96259911996461e-05, "loss": 0.024426670745015144, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02473, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 2.0028319358825684, "learning_rate": 9.958771393851491e-05, "loss": 0.04147466644644737, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.04235, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 0.7103351950645447, "learning_rate": 9.954758120060702e-05, "loss": 0.027535825967788696, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02792, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 7.31724739074707, "learning_rate": 9.950559465600948e-05, "loss": 0.030773598700761795, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.03125, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 0.7934699654579163, "learning_rate": 9.946175605195379e-05, "loss": 0.02381661906838417, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.0241, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.8, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 2.041188955307007, "learning_rate": 9.941606721274322e-05, "loss": 0.033892303705215454, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.03447, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 0.8157190680503845, "learning_rate": 9.936853003967685e-05, "loss": 0.015917936339974403, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01605, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 1.525278091430664, "learning_rate": 9.93191465109705e-05, "loss": 0.03661094978451729, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.03729, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.4628369808197021, "learning_rate": 9.926791868167438e-05, "loss": 0.04371285066008568, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.04468, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 0.693761944770813, "learning_rate": 9.921484868358753e-05, "loss": 0.03235514089465141, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.03288, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.6666591167449951, "learning_rate": 9.915993872516924e-05, "loss": 0.018807969987392426, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01899, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 7.3516130447387695, "learning_rate": 9.9103191091447e-05, "loss": 0.017487727105617523, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01764, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 1.2322312593460083, "learning_rate": 9.904460814392147e-05, "loss": 0.030732661485671997, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03121, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.47, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.7826405763626099, "learning_rate": 9.898419232046825e-05, "loss": 0.028215378522872925, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02862, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.6134302020072937, "learning_rate": 9.892194613523633e-05, "loss": 0.021848907694220543, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.02209, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 1.1514124870300293, "learning_rate": 9.885787217854357e-05, "loss": 0.020309554412961006, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.02052, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.4455357789993286, "learning_rate": 9.879197311676887e-05, "loss": 0.016322335228323936, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01646, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 3.4006857872009277, "learning_rate": 9.872425169224113e-05, "loss": 0.028112702071666718, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.02851, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.9206350445747375, "learning_rate": 9.865471072312528e-05, "loss": 0.02538941241800785, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02571, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.2755025625228882, "learning_rate": 9.858335310330492e-05, "loss": 0.0044681658037006855, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00448, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.9982885718345642, "learning_rate": 9.851018180226185e-05, "loss": 0.04370322823524475, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04467, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.9355953335762024, "learning_rate": 9.843519986495259e-05, "loss": 0.030878858640789986, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.03136, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 1.2075616121292114, "learning_rate": 9.835841041168162e-05, "loss": 0.017268726602196693, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01742, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 1.6571303606033325, "learning_rate": 9.82798166379715e-05, "loss": 0.012560022994875908, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01264, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 1.3167718648910522, "learning_rate": 9.819942181443002e-05, "loss": 0.00814415980130434, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00818, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 0.7375222444534302, "learning_rate": 9.811722928661392e-05, "loss": 0.037614770233631134, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.03833, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 0.20292189717292786, "learning_rate": 9.803324247488975e-05, "loss": 0.004641775973141193, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00465, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.6306488513946533, "learning_rate": 9.794746487429161e-05, "loss": 0.009082251228392124, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.00912, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.976203203201294, "learning_rate": 9.785990005437554e-05, "loss": 0.018044915050268173, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01821, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 1.8152360916137695, "learning_rate": 9.777055165907117e-05, "loss": 0.017329659312963486, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.01748, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 1.5518664121627808, "learning_rate": 9.767942340652993e-05, "loss": 0.027958203107118607, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02835, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 1.45272696018219, "learning_rate": 9.758651908897035e-05, "loss": 0.03647725284099579, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.03715, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.2473431825637817, "learning_rate": 9.749184257252033e-05, "loss": 0.040240682661533356, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.04106, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.8617854118347168, "learning_rate": 9.739539779705614e-05, "loss": 0.014753181487321854, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01486, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.5910308957099915, "learning_rate": 9.729718877603861e-05, "loss": 0.016973398625850677, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01712, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.8269942998886108, "learning_rate": 9.719721959634592e-05, "loss": 0.030902016907930374, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03138, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.5857502222061157, "learning_rate": 9.709549441810375e-05, "loss": 0.012737656943500042, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01282, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.7836717367172241, "learning_rate": 9.699201747451195e-05, "loss": 0.03418467193841934, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.03478, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.6360130310058594, "learning_rate": 9.688679307166854e-05, "loss": 0.019799165427684784, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.02, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.7259521484375, "learning_rate": 9.677982558839042e-05, "loss": 0.02308635786175728, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02335, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.6470049619674683, "learning_rate": 9.66711194760312e-05, "loss": 0.010334879159927368, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01039, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 1.3848823308944702, "learning_rate": 9.656067925829593e-05, "loss": 0.038049183785915375, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.03878, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.9381147623062134, "learning_rate": 9.644850953105288e-05, "loss": 0.020919155329465866, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.02114, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.7758721709251404, "learning_rate": 9.633461496214225e-05, "loss": 0.03495250642299652, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.03557, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.775175154209137, "learning_rate": 9.621900029118195e-05, "loss": 0.034823838621377945, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03544, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.3491177260875702, "learning_rate": 9.610167032937036e-05, "loss": 0.006522779352962971, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00654, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.6705529689788818, "learning_rate": 9.598262995928611e-05, "loss": 0.023599062114953995, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02388, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.23, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.3620026707649231, "learning_rate": 9.586188413468492e-05, "loss": 0.014357725158333778, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01446, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.5337263941764832, "learning_rate": 9.57394378802934e-05, "loss": 0.028328459709882736, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.02873, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 0.6535420417785645, "learning_rate": 9.56152962916e-05, "loss": 0.035336896777153015, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.03597, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.37656787037849426, "learning_rate": 9.548946453464296e-05, "loss": 0.009073600172996521, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00911, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.37074121832847595, "learning_rate": 9.53619478457953e-05, "loss": 0.01729549653828144, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01745, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.47772303223609924, "learning_rate": 9.523275153154695e-05, "loss": 0.015557825565338135, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01568, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.6403437256813049, "learning_rate": 9.51018809682839e-05, "loss": 0.03662584722042084, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0373, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.44938117265701294, "learning_rate": 9.49693416020645e-05, "loss": 0.02172817848622799, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.02197, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.3531762659549713, "learning_rate": 9.483513894839276e-05, "loss": 0.008097702637314796, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00813, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.44762036204338074, "learning_rate": 9.469927859198888e-05, "loss": 0.009483933448791504, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00953, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.8377417325973511, "learning_rate": 9.456176618655689e-05, "loss": 0.021944141015410423, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02219, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 3.2718989849090576, "learning_rate": 9.442260745454927e-05, "loss": 0.04655059799551964, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.04765, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.5034907460212708, "learning_rate": 9.428180818692884e-05, "loss": 0.009491579607129097, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00954, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.4602927267551422, "learning_rate": 9.413937424292791e-05, "loss": 0.009914261288940907, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00996, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.61, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.46201199293136597, "learning_rate": 9.399531154980424e-05, "loss": 0.00961692351847887, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.00966, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.5251612663269043, "learning_rate": 9.384962610259455e-05, "loss": 0.006039257161319256, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00606, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.2495802342891693, "learning_rate": 9.370232396386494e-05, "loss": 0.0031449247617274523, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00315, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.34412190318107605, "learning_rate": 9.355341126345868e-05, "loss": 0.007052200380712748, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00708, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 1.284787654876709, "learning_rate": 9.340289419824107e-05, "loss": 0.01793455332517624, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.0181, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.22492077946662903, "learning_rate": 9.325077903184159e-05, "loss": 0.004193002823740244, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0042, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.12834703922271729, "learning_rate": 9.30970720943932e-05, "loss": 0.00117142079398036, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00117, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.7376601099967957, "learning_rate": 9.2941779782269e-05, "loss": 0.01673451066017151, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01688, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 0.6762248873710632, "learning_rate": 9.278490855781596e-05, "loss": 0.012961724773049355, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01305, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 30.97, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 0.9987324476242065, "learning_rate": 9.262646494908604e-05, "loss": 0.019328705966472626, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01952, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 0.4835776686668396, "learning_rate": 9.246645554956457e-05, "loss": 0.008623412810266018, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00866, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.9454216361045837, "learning_rate": 9.230488701789578e-05, "loss": 0.008428583852946758, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00846, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.42290356755256653, "learning_rate": 9.214176607760577e-05, "loss": 0.009121299721300602, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00916, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 1.537367582321167, "learning_rate": 9.197709951682268e-05, "loss": 0.013702074997127056, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.0138, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 1.4272860288619995, "learning_rate": 9.181089418799428e-05, "loss": 0.02061416208744049, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.02083, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.8676391243934631, "learning_rate": 9.164315700760271e-05, "loss": 0.01789386011660099, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01805, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.4209684729576111, "learning_rate": 9.147389495587671e-05, "loss": 0.00799693912267685, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00803, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.7882896065711975, "learning_rate": 9.130311507650116e-05, "loss": 0.02348099648952484, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.02376, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.7079359292984009, "learning_rate": 9.113082447632394e-05, "loss": 0.019334180280566216, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01952, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 32.77, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.6180799007415771, "learning_rate": 9.09570303250602e-05, "loss": 0.01295492984354496, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01304, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.3, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.31305569410324097, "learning_rate": 9.078173985499394e-05, "loss": 0.007109765894711018, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00714, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.7431589365005493, "learning_rate": 9.060496036067713e-05, "loss": 0.020079202950000763, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02028, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.9217796325683594, "learning_rate": 9.042669919862615e-05, "loss": 0.025663159787654877, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.026, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.31291723251342773, "learning_rate": 9.024696378701557e-05, "loss": 0.010684587061405182, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01074, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.61, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.4710655212402344, "learning_rate": 9.006576160536948e-05, "loss": 0.01782049797475338, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01798, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 1.6839576959609985, "learning_rate": 8.988310019425035e-05, "loss": 0.010420205071568489, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01047, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.8137030005455017, "learning_rate": 8.969898715494506e-05, "loss": 0.023721465840935707, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02401, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 0.4625418186187744, "learning_rate": 8.951343014914869e-05, "loss": 0.013024603947997093, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01311, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.11397802084684372, "learning_rate": 8.932643689864568e-05, "loss": 0.0023826193064451218, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00239, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.34063583612442017, "learning_rate": 8.913801518498845e-05, "loss": 0.01078672707080841, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01085, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.12, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.46385636925697327, "learning_rate": 8.894817284917364e-05, "loss": 0.009202694520354271, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00925, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.4294778108596802, "learning_rate": 8.875691779131569e-05, "loss": 0.015953533351421356, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01608, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.56, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 1.0608989000320435, "learning_rate": 8.856425797031829e-05, "loss": 0.013293517753481865, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.01338, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 1.3682215213775635, "learning_rate": 8.837020140354295e-05, "loss": 0.02662494033575058, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.02698, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.30824294686317444, "learning_rate": 8.817475616647554e-05, "loss": 0.01093093492090702, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01099, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.11879246681928635, "learning_rate": 8.797793039239017e-05, "loss": 0.0024915540125221014, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00249, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.43917909264564514, "learning_rate": 8.777973227201069e-05, "loss": 0.009152824990451336, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00919, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.4326302409172058, "learning_rate": 8.758017005316988e-05, "loss": 0.01793557032942772, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.0181, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 0.3883635401725769, "learning_rate": 8.737925204046629e-05, "loss": 0.007490199990570545, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00752, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.7711528539657593, "learning_rate": 8.717698659491851e-05, "loss": 0.009999990463256836, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01005, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 1.4223685264587402, "learning_rate": 8.697338213361735e-05, "loss": 0.019633794203400612, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01983, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.3366284668445587, "learning_rate": 8.676844712937552e-05, "loss": 0.011083938181400299, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.01115, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.472940593957901, "learning_rate": 8.656219011037509e-05, "loss": 0.012749402783811092, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01283, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.29737389087677, "learning_rate": 8.63546196598125e-05, "loss": 0.006160742603242397, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00618, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.6953185796737671, "learning_rate": 8.614574441554145e-05, "loss": 0.02069683000445366, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.02091, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.9643389582633972, "learning_rate": 8.593557306971349e-05, "loss": 0.01692148484289646, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01707, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.44834357500076294, "learning_rate": 8.572411436841618e-05, "loss": 0.0134150804951787, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01351, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.5362644195556641, "learning_rate": 8.551137711130922e-05, "loss": 0.011906599625945091, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.01198, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.5097058415412903, "learning_rate": 8.529737015125824e-05, "loss": 0.018725916743278503, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0189, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.26728132367134094, "learning_rate": 8.508210239396639e-05, "loss": 0.008682526648044586, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00872, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.1433325707912445, "learning_rate": 8.486558279760375e-05, "loss": 0.0033757323399186134, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00338, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.2423364371061325, "learning_rate": 8.464782037243449e-05, "loss": 0.0077317566610872746, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00776, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.2378045618534088, "learning_rate": 8.442882418044202e-05, "loss": 0.009452993981540203, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0095, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 0.5414705276489258, "learning_rate": 8.420860333495179e-05, "loss": 0.01184141170233488, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01191, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.3702496588230133, "learning_rate": 8.398716700025208e-05, "loss": 0.005873196758329868, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00589, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 1.6058319807052612, "learning_rate": 8.376452439121266e-05, "loss": 0.045116156339645386, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.04615, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.53, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 1.4317865371704102, "learning_rate": 8.354068477290124e-05, "loss": 0.007007307838648558, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00703, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.2702924311161041, "learning_rate": 8.331565746019807e-05, "loss": 0.004837811924517155, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00485, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.7970258593559265, "learning_rate": 8.308945181740812e-05, "loss": 0.024860447272658348, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.02517, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.37202033400535583, "learning_rate": 8.286207725787153e-05, "loss": 0.009003996849060059, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00904, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 0.6545891761779785, "learning_rate": 8.263354324357182e-05, "loss": 0.015379521995782852, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0155, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.6331518888473511, "learning_rate": 8.240385928474219e-05, "loss": 0.0064120120368897915, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00643, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.2593384385108948, "learning_rate": 8.217303493946967e-05, "loss": 0.00396647397428751, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00397, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.4497303366661072, "learning_rate": 8.194107981329746e-05, "loss": 0.01667815074324608, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01682, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.3270517885684967, "learning_rate": 8.170800355882518e-05, "loss": 0.0016992766177281737, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.0017, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 0.7203754782676697, "learning_rate": 8.147381587530713e-05, "loss": 0.0034559094347059727, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00346, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 2.6674413681030273, "learning_rate": 8.123852650824877e-05, "loss": 0.024432891979813576, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02473, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 0.9376115202903748, "learning_rate": 8.100214524900103e-05, "loss": 0.03528985753655434, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.03592, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 22.08, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.18790878355503082, "learning_rate": 8.076468193435301e-05, "loss": 0.003229245776310563, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00323, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.22934910655021667, "learning_rate": 8.052614644612253e-05, "loss": 0.002949900459498167, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00295, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.42834317684173584, "learning_rate": 8.028654871074489e-05, "loss": 0.006748121231794357, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00677, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.31441730260849, "learning_rate": 8.004589869885986e-05, "loss": 0.004972625989466906, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00499, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.7083582878112793, "learning_rate": 7.980420642489674e-05, "loss": 0.009047096595168114, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00909, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.12135189026594162, "learning_rate": 7.95614819466576e-05, "loss": 0.0021659513004124165, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00217, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 1.6076438426971436, "learning_rate": 7.931773536489872e-05, "loss": 0.04110487177968025, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.04196, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 3.217456102371216, "learning_rate": 7.907297682291035e-05, "loss": 0.020579630509018898, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02079, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.3666270077228546, "learning_rate": 7.882721650609442e-05, "loss": 0.014149104245007038, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01425, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.7760597467422485, "learning_rate": 7.85804646415409e-05, "loss": 0.018994729965925217, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01918, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.19938381016254425, "learning_rate": 7.833273149760207e-05, "loss": 0.003920767456293106, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00393, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.7, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 1.1444849967956543, "learning_rate": 7.808402738346527e-05, "loss": 0.03356565162539482, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03414, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.2678207457065582, "learning_rate": 7.783436264872382e-05, "loss": 0.005675951950252056, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00569, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.29650187492370605, "learning_rate": 7.758374768294647e-05, "loss": 0.007949892431497574, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00798, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.4538792669773102, "learning_rate": 7.733219291524489e-05, "loss": 0.013670050539076328, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01376, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.36740994453430176, "learning_rate": 7.707970881383977e-05, "loss": 0.007569313980638981, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.0076, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 28.91, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.5982331037521362, "learning_rate": 7.682630588562518e-05, "loss": 0.014960374683141708, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01507, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.7214632630348206, "learning_rate": 7.657199467573129e-05, "loss": 0.01106884703040123, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01113, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.5613954067230225, "learning_rate": 7.631678576708561e-05, "loss": 0.00951668992638588, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00956, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.5739856958389282, "learning_rate": 7.606068977997255e-05, "loss": 0.010577734559774399, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.01063, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 1.5730392932891846, "learning_rate": 7.580371737159148e-05, "loss": 0.0079564917832613, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00799, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.07926855981349945, "learning_rate": 7.554587923561324e-05, "loss": 0.0010473887668922544, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00105, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 1.7014634609222412, "learning_rate": 7.528718610173511e-05, "loss": 0.026765871793031693, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02713, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.69, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.36417582631111145, "learning_rate": 7.502764873523431e-05, "loss": 0.005272375885397196, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00529, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.7487906217575073, "learning_rate": 7.476727793652011e-05, "loss": 0.014359857887029648, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.01446, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.7016041874885559, "learning_rate": 7.450608454068415e-05, "loss": 0.0172574520111084, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01741, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.4716252088546753, "learning_rate": 7.424407941704987e-05, "loss": 0.009377402253448963, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00942, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.49418941140174866, "learning_rate": 7.398127346871986e-05, "loss": 0.015757262706756592, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01588, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.1043359711766243, "learning_rate": 7.371767763212238e-05, "loss": 0.0016512771835550666, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00165, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.24825944006443024, "learning_rate": 7.345330287655617e-05, "loss": 0.0057617174461483955, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00578, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.57, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.7858842611312866, "learning_rate": 7.31881602037339e-05, "loss": 0.024054482579231262, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02435, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.37271323800086975, "learning_rate": 7.29222606473245e-05, "loss": 0.009181841276586056, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00922, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.3700115978717804, "learning_rate": 7.265561527249383e-05, "loss": 0.009475287050008774, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00952, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 3.4071478843688965, "learning_rate": 7.238823517544436e-05, "loss": 0.025362709537148476, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.02569, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.2977069914340973, "learning_rate": 7.212013148295333e-05, "loss": 0.0037982354406267405, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00381, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.57, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.2025032639503479, "learning_rate": 7.185131535190975e-05, "loss": 0.006078783888369799, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0061, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 29.79, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 1.6224626302719116, "learning_rate": 7.158179796885005e-05, "loss": 0.005352355074137449, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00537, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.18762408196926117, "learning_rate": 7.131159054949273e-05, "loss": 0.004200303927063942, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00421, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.2, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.5216138958930969, "learning_rate": 7.104070433827139e-05, "loss": 0.010421667248010635, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01048, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.82, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.14681372046470642, "learning_rate": 7.076915060786705e-05, "loss": 0.002897534053772688, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0029, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.6473350524902344, "learning_rate": 7.049694065873882e-05, "loss": 0.0058282348327338696, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00585, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.7301570177078247, "learning_rate": 7.022408581865382e-05, "loss": 0.010120212100446224, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01017, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.14110901951789856, "learning_rate": 6.99505974422157e-05, "loss": 0.0012285812990739942, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00123, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.09392605721950531, "learning_rate": 6.967648691039213e-05, "loss": 0.0013195722131058574, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00132, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.3541688919067383, "learning_rate": 6.940176563004123e-05, "loss": 0.010699443519115448, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01076, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.27358636260032654, "learning_rate": 6.912644503343682e-05, "loss": 0.003917871508747339, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00393, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.7341279983520508, "learning_rate": 6.885053657779273e-05, "loss": 0.02089763432741165, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02112, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.24617047607898712, "learning_rate": 6.857405174478604e-05, "loss": 0.003145552473142743, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00315, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.6437750458717346, "learning_rate": 6.82970020400792e-05, "loss": 0.02655647322535515, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02691, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.3027588427066803, "learning_rate": 6.801939899284132e-05, "loss": 0.004079680889844894, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00409, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.08927033841609955, "learning_rate": 6.774125415526827e-05, "loss": 0.0015130606479942799, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00151, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.024218542501330376, "learning_rate": 6.746257910210214e-05, "loss": 0.00024706448311917484, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00025, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.44474026560783386, "learning_rate": 6.718338543014937e-05, "loss": 0.007681374438107014, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00771, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.44944825768470764, "learning_rate": 6.69036847577983e-05, "loss": 0.0075889998115599155, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00762, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.7835424542427063, "learning_rate": 6.662348872453553e-05, "loss": 0.022515609860420227, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.02277, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.15066367387771606, "learning_rate": 6.63428089904618e-05, "loss": 0.0034368583001196384, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00344, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.11268172413110733, "learning_rate": 6.60616572358065e-05, "loss": 0.002663365565240383, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00267, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.211436465382576, "learning_rate": 6.578004516044172e-05, "loss": 0.001301605487242341, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0013, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.1382550150156021, "learning_rate": 6.549798448339548e-05, "loss": 0.002714708214625716, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00272, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.12282419949769974, "learning_rate": 6.521548694236384e-05, "loss": 0.0012893605744466186, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00129, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.7146168351173401, "learning_rate": 6.493256429322259e-05, "loss": 0.02380497194826603, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.02409, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.04, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.3827158212661743, "learning_rate": 6.464922830953799e-05, "loss": 0.021427078172564507, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.02166, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.13, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 1.920496940612793, "learning_rate": 6.436549078207688e-05, "loss": 0.025075701996684074, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02539, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.45099982619285583, "learning_rate": 6.408136351831592e-05, "loss": 0.016066597774624825, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.0162, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.20386943221092224, "learning_rate": 6.379685834195036e-05, "loss": 0.004237259738147259, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00425, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.16201196610927582, "learning_rate": 6.351198709240186e-05, "loss": 0.002916014287620783, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00292, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.83, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.27638620138168335, "learning_rate": 6.32267616243259e-05, "loss": 0.011516126804053783, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.01158, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.19703803956508636, "learning_rate": 6.294119380711849e-05, "loss": 0.006947541609406471, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00697, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.5232514142990112, "learning_rate": 6.265529552442209e-05, "loss": 0.019201086834073067, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01939, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.17307600378990173, "learning_rate": 6.236907867363127e-05, "loss": 0.004894524812698364, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00491, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.32801973819732666, "learning_rate": 6.208255516539749e-05, "loss": 0.00896113459020853, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.009, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.12270452827215195, "learning_rate": 6.179573692313344e-05, "loss": 0.0030153580009937286, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00302, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.24035434424877167, "learning_rate": 6.150863588251694e-05, "loss": 0.007372635882347822, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0074, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.16784679889678955, "learning_rate": 6.122126399099419e-05, "loss": 0.0047536930069327354, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00477, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.5763794183731079, "learning_rate": 6.0933633207282615e-05, "loss": 0.020335718989372253, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.02054, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.1585373431444168, "learning_rate": 6.064575550087316e-05, "loss": 0.0052315727807581425, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00525, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.17109814286231995, "learning_rate": 6.0357642851532245e-05, "loss": 0.0030386471189558506, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00304, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.412850558757782, "learning_rate": 6.0069307248803294e-05, "loss": 0.021409496665000916, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.02164, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.72, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.29618701338768005, "learning_rate": 5.9780760691507635e-05, "loss": 0.00824391283094883, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00828, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.46008098125457764, "learning_rate": 5.9492015187245334e-05, "loss": 0.005213115364313126, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00523, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.08966958522796631, "learning_rate": 5.920308275189541e-05, "loss": 0.0008495047804899514, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00085, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.27686089277267456, "learning_rate": 5.8913975409115874e-05, "loss": 0.008544478565454483, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00858, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.8912631869316101, "learning_rate": 5.8624705189843395e-05, "loss": 0.005141790024936199, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00516, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 0.05179021134972572, "learning_rate": 5.833528413179249e-05, "loss": 0.0005013138870708644, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.0005, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.10541129857301712, "learning_rate": 5.80457242789548e-05, "loss": 0.0012643830850720406, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00127, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.38575249910354614, "learning_rate": 5.77560376810977e-05, "loss": 0.005144515540450811, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00516, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.5090954899787903, "learning_rate": 5.7466236393263005e-05, "loss": 0.0035987270530313253, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00361, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.031204959377646446, "learning_rate": 5.717633247526522e-05, "loss": 0.0003923364565707743, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00039, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.15446174144744873, "learning_rate": 5.688633799118971e-05, "loss": 0.0011735681910067797, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00117, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.2854711413383484, "learning_rate": 5.659626500889066e-05, "loss": 0.005174555815756321, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00519, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.3126105070114136, "learning_rate": 5.6306125599488905e-05, "loss": 0.0026795240119099617, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00268, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 31.76, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.2406930923461914, "learning_rate": 5.601593183686955e-05, "loss": 0.002271719742566347, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00227, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.3227218985557556, "learning_rate": 5.572569579717961e-05, "loss": 0.014228536747395992, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.01433, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.262531578540802, "learning_rate": 5.543542955832538e-05, "loss": 0.00418767798691988, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0042, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.43, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.29595431685447693, "learning_rate": 5.514514519946986e-05, "loss": 0.004800382535904646, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00481, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.28463131189346313, "learning_rate": 5.485485480053015e-05, "loss": 0.0026832744479179382, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00269, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.36745983362197876, "learning_rate": 5.4564570441674645e-05, "loss": 0.0020881728269159794, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00209, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.11928800493478775, "learning_rate": 5.42743042028204e-05, "loss": 0.0012180630583316088, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00122, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.75, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.1157614067196846, "learning_rate": 5.3984068163130464e-05, "loss": 0.0011398275382816792, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00114, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.623539924621582, "learning_rate": 5.369387440051111e-05, "loss": 0.006659184582531452, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00668, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.05786527320742607, "learning_rate": 5.340373499110935e-05, "loss": 0.00047375919530168176, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00047, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.008514405228197575, "learning_rate": 5.3113662008810304e-05, "loss": 0.0001492338051320985, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00015, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.09148407727479935, "learning_rate": 5.282366752473479e-05, "loss": 0.000862723623868078, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00086, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.8457103967666626, "learning_rate": 5.2533763606737005e-05, "loss": 0.01685916632413864, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.017, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.18, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.5437982082366943, "learning_rate": 5.224396231890232e-05, "loss": 0.008787979371845722, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00883, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.42195308208465576, "learning_rate": 5.195427572104522e-05, "loss": 0.008496478199958801, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00853, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.4132539927959442, "learning_rate": 5.166471586820751e-05, "loss": 0.002860720269382, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00286, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.39163756370544434, "learning_rate": 5.1375294810156615e-05, "loss": 0.022039305418729782, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02228, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.7006239295005798, "learning_rate": 5.1086024590884144e-05, "loss": 0.008574279025197029, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00861, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.31496870517730713, "learning_rate": 5.079691724810461e-05, "loss": 0.0012693319004029036, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00127, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.63, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.12253858894109726, "learning_rate": 5.0507984812754684e-05, "loss": 0.0022763900924474, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00228, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.4226369559764862, "learning_rate": 5.021923930849237e-05, "loss": 0.008194214664399624, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00823, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.11040602624416351, "learning_rate": 4.99306927511967e-05, "loss": 0.002819106914103031, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00282, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.042430996894836426, "learning_rate": 4.964235714846775e-05, "loss": 0.0008261682232841849, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00083, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.11226201802492142, "learning_rate": 4.9354244499126866e-05, "loss": 0.0019186226418241858, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00192, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.21, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.08060327917337418, "learning_rate": 4.90663667927174e-05, "loss": 0.0009203572990372777, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00092, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.69, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.22906987369060516, "learning_rate": 4.877873600900581e-05, "loss": 0.0027872640639543533, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00279, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.57, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.4380970299243927, "learning_rate": 4.849136411748306e-05, "loss": 0.0020403899252414703, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00204, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.2687428891658783, "learning_rate": 4.8204263076866574e-05, "loss": 0.004954529460519552, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00497, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.43, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.16702799499034882, "learning_rate": 4.791744483460251e-05, "loss": 0.0025928220711648464, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0026, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.16557523608207703, "learning_rate": 4.7630921326368736e-05, "loss": 0.0035158060491085052, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00352, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 19.91, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.1214073896408081, "learning_rate": 4.7344704475577916e-05, "loss": 0.0014897201908752322, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00149, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.3516741991043091, "learning_rate": 4.705880619288153e-05, "loss": 0.0075050522573292255, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00753, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 21.89, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.16479502618312836, "learning_rate": 4.677323837567412e-05, "loss": 0.0024614338763058186, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00246, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.0868649110198021, "learning_rate": 4.6488012907598146e-05, "loss": 0.0011970661580562592, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.0012, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.2552068531513214, "learning_rate": 4.620314165804964e-05, "loss": 0.00400127749890089, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00401, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.07049008458852768, "learning_rate": 4.591863648168407e-05, "loss": 0.0008490943582728505, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00085, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.32708409428596497, "learning_rate": 4.5634509217923135e-05, "loss": 0.002135287504643202, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00214, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.0248547475785017, "learning_rate": 4.535077169046201e-05, "loss": 0.00036402817931957543, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00036, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.46936774253845215, "learning_rate": 4.506743570677743e-05, "loss": 0.007857133634388447, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00789, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.05, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.37595105171203613, "learning_rate": 4.478451305763618e-05, "loss": 0.004991983063519001, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.005, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.36, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.3728984296321869, "learning_rate": 4.450201551660454e-05, "loss": 0.010508553124964237, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.01056, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.8831390142440796, "learning_rate": 4.4219954839558276e-05, "loss": 0.018092596903443336, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01826, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.24824120104312897, "learning_rate": 4.393834276419352e-05, "loss": 0.004578567575663328, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00459, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.15880343317985535, "learning_rate": 4.36571910095382e-05, "loss": 0.0006069833762012422, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00061, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.13295789062976837, "learning_rate": 4.337651127546448e-05, "loss": 0.0018690828001126647, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00187, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.14602690935134888, "learning_rate": 4.3096315242201736e-05, "loss": 0.0013887430541217327, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00139, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.23219968378543854, "learning_rate": 4.2816614569850635e-05, "loss": 0.0030695421155542135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00307, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.25426629185676575, "learning_rate": 4.2537420897897864e-05, "loss": 0.0023034540936350822, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00231, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.013862662948668003, "learning_rate": 4.225874584473174e-05, "loss": 0.00015099707525223494, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00015, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.03588486090302467, "learning_rate": 4.19806010071587e-05, "loss": 0.0003493187832646072, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00035, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.44930118322372437, "learning_rate": 4.170299795992081e-05, "loss": 0.007733460515737534, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00776, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.12348396331071854, "learning_rate": 4.142594825521398e-05, "loss": 0.0009035887196660042, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.0009, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.01762501150369644, "learning_rate": 4.114946342220728e-05, "loss": 0.0001744318869896233, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00017, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.3107796907424927, "learning_rate": 4.087355496656321e-05, "loss": 0.002448304323479533, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00245, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.06949736177921295, "learning_rate": 4.05982343699588e-05, "loss": 0.0010190955363214016, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00102, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.020290974527597427, "learning_rate": 4.0323513089607876e-05, "loss": 0.0002621181192807853, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00026, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.8925724625587463, "learning_rate": 4.004940255778431e-05, "loss": 0.01524729747325182, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01536, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.03912007436156273, "learning_rate": 3.977591418134619e-05, "loss": 0.0005216171266511083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00052, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.13822810351848602, "learning_rate": 3.95030593412612e-05, "loss": 0.002425316022709012, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00243, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.15779589116573334, "learning_rate": 3.923084939213296e-05, "loss": 0.0010419909376651049, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00104, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.3634334206581116, "learning_rate": 3.895929566172861e-05, "loss": 0.005640519317239523, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00566, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 1.0640664100646973, "learning_rate": 3.868840945050728e-05, "loss": 0.010304893366992474, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.01036, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.43, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.5768812298774719, "learning_rate": 3.841820203114995e-05, "loss": 0.012604037299752235, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.01268, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.04282154515385628, "learning_rate": 3.814868464809027e-05, "loss": 0.0004884580848738551, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00049, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.347759872674942, "learning_rate": 3.787986851704667e-05, "loss": 0.007089672610163689, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00711, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.02373470366001129, "learning_rate": 3.7611764824555654e-05, "loss": 0.00021606020163744688, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00022, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.3379289507865906, "learning_rate": 3.734438472750619e-05, "loss": 0.0028585337568074465, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00286, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.04415091127157211, "learning_rate": 3.707773935267552e-05, "loss": 0.0005585376638919115, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00056, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.12732049822807312, "learning_rate": 3.68118397962661e-05, "loss": 0.0014658442232757807, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00147, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.022468917071819305, "learning_rate": 3.654669712344384e-05, "loss": 0.00018597961752675474, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00019, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.01311606913805008, "learning_rate": 3.628232236787763e-05, "loss": 0.00013859791215509176, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00014, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.20352588593959808, "learning_rate": 3.6018726531280144e-05, "loss": 0.0027597742155194283, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00276, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.2692115008831024, "learning_rate": 3.575592058295017e-05, "loss": 0.0025300646666437387, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00253, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.018283380195498466, "learning_rate": 3.549391545931585e-05, "loss": 0.00022599783551413566, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00023, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.0767168402671814, "learning_rate": 3.5232722063479914e-05, "loss": 0.0006083987536840141, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00061, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.014371824450790882, "learning_rate": 3.49723512647657e-05, "loss": 0.0002593999379314482, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00026, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.15987667441368103, "learning_rate": 3.471281389826491e-05, "loss": 0.0013809651136398315, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00138, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.24991489946842194, "learning_rate": 3.4454120764386764e-05, "loss": 0.00031548645347356796, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00032, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.21022218465805054, "learning_rate": 3.4196282628408526e-05, "loss": 0.0014502726262435317, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00145, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.07417729496955872, "learning_rate": 3.3939310220027456e-05, "loss": 0.0006328061572276056, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00063, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.11230316758155823, "learning_rate": 3.3683214232914404e-05, "loss": 0.0015101988101378083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00151, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.015879293903708458, "learning_rate": 3.342800532426873e-05, "loss": 0.00013681515702046454, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00014, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.23006685078144073, "learning_rate": 3.317369411437484e-05, "loss": 0.0017989645712077618, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.0018, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.07562465220689774, "learning_rate": 3.292029118616024e-05, "loss": 0.000738563307095319, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00074, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.4829797148704529, "learning_rate": 3.266780708475511e-05, "loss": 0.004725145176053047, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00474, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.027833323925733566, "learning_rate": 3.241625231705354e-05, "loss": 0.0002455330395605415, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00025, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.02027474157512188, "learning_rate": 3.216563735127618e-05, "loss": 0.00020762838539667428, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00021, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.39427897334098816, "learning_rate": 3.191597261653475e-05, "loss": 0.005458880215883255, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00547, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 0.12294086068868637, "learning_rate": 3.166726850239794e-05, "loss": 0.001134907710365951, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00114, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.003537102835252881, "learning_rate": 3.141953535845912e-05, "loss": 5.5187840189319104e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00006, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.24, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.12407265603542328, "learning_rate": 3.11727834939056e-05, "loss": 0.0004565584531519562, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00046, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.2576903998851776, "learning_rate": 3.092702317708967e-05, "loss": 0.01024781446903944, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0103, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.00871451385319233, "learning_rate": 3.0682264635101276e-05, "loss": 0.0001011230779113248, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0001, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.18038737773895264, "learning_rate": 3.0438518053342407e-05, "loss": 0.0021058323327451944, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00211, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.43194493651390076, "learning_rate": 3.0195793575103266e-05, "loss": 0.004028185270726681, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00404, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.15, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.19127167761325836, "learning_rate": 2.9954101301140146e-05, "loss": 0.0033325315453112125, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00334, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.028812039643526077, "learning_rate": 2.9713451289255123e-05, "loss": 0.000239394954405725, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00024, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.01365281268954277, "learning_rate": 2.9473853553877484e-05, "loss": 0.00016896855959203094, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00017, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.16845834255218506, "learning_rate": 2.9235318065647e-05, "loss": 0.001374136540107429, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00138, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.05288142338395119, "learning_rate": 2.8997854750998964e-05, "loss": 0.00047174817882478237, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00047, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 23.81, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.2515280544757843, "learning_rate": 2.8761473491751258e-05, "loss": 0.0010539848590269685, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00105, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.9719142317771912, "learning_rate": 2.8526184124692883e-05, "loss": 0.0082389609888196, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00827, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.29, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.0369810052216053, "learning_rate": 2.829199644117484e-05, "loss": 0.0003323935379739851, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00033, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.16510939598083496, "learning_rate": 2.8058920186702553e-05, "loss": 0.0011406682897359133, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00114, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.09079772233963013, "learning_rate": 2.782696506053033e-05, "loss": 0.0008543305448256433, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00085, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.24, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.45489054918289185, "learning_rate": 2.7596140715257824e-05, "loss": 0.0028872634284198284, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00289, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.01444367878139019, "learning_rate": 2.7366456756428184e-05, "loss": 0.0001377114385832101, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00014, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.052114181220531464, "learning_rate": 2.7137922742128486e-05, "loss": 0.00025695108342915773, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00026, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.036422811448574066, "learning_rate": 2.691054818259188e-05, "loss": 0.0003630987775977701, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00036, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 1.3374916315078735, "learning_rate": 2.6684342539801933e-05, "loss": 0.004096651449799538, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00411, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.44288432598114014, "learning_rate": 2.645931522709877e-05, "loss": 0.007123283110558987, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00715, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.40779948234558105, "learning_rate": 2.6235475608787365e-05, "loss": 0.004187433049082756, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.0042, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.3204880356788635, "learning_rate": 2.6012832999747916e-05, "loss": 0.002053872449323535, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00206, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.39905402064323425, "learning_rate": 2.579139666504821e-05, "loss": 0.014745216816663742, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01485, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.13356807827949524, "learning_rate": 2.557117581955798e-05, "loss": 0.0010441134218126535, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00104, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.04006842523813248, "learning_rate": 2.5352179627565532e-05, "loss": 0.0004148203879594803, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00041, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.2026113122701645, "learning_rate": 2.5134417202396277e-05, "loss": 0.002482743700966239, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00249, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.09398742020130157, "learning_rate": 2.491789760603361e-05, "loss": 0.0011718376772478223, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00117, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.01526392437517643, "learning_rate": 2.4702629848741764e-05, "loss": 0.00016991692245937884, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00017, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.5203941464424133, "learning_rate": 2.4488622888690785e-05, "loss": 0.01559620164334774, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.01572, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.019999055191874504, "learning_rate": 2.427588563158384e-05, "loss": 0.0002613818214740604, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00026, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.11791342496871948, "learning_rate": 2.406442693028651e-05, "loss": 0.001244979677721858, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00125, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.010082939639687538, "learning_rate": 2.3854255584458547e-05, "loss": 0.00019661366241052747, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0002, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.48, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.31394556164741516, "learning_rate": 2.3645380340187508e-05, "loss": 0.0029331324622035027, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00294, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.12033931910991669, "learning_rate": 2.3437809889624914e-05, "loss": 0.0017682103207334876, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00177, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.0163614209741354, "learning_rate": 2.3231552870624487e-05, "loss": 0.00021498536807484925, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00022, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.044644154608249664, "learning_rate": 2.3026617866382657e-05, "loss": 0.0005792760639451444, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00058, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.74, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.013698357157409191, "learning_rate": 2.2823013405081507e-05, "loss": 0.0002790935686789453, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00028, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.07949245721101761, "learning_rate": 2.2620747959533722e-05, "loss": 0.0007771623786538839, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00078, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.54, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.08225737512111664, "learning_rate": 2.2419829946830123e-05, "loss": 0.0009477273561060429, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00095, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.1294616162776947, "learning_rate": 2.2220267727989325e-05, "loss": 0.0024821748957037926, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00249, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.18926918506622314, "learning_rate": 2.202206960760984e-05, "loss": 0.003097239648923278, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0031, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.11995252221822739, "learning_rate": 2.182524383352446e-05, "loss": 0.00128280115313828, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00128, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.18343648314476013, "learning_rate": 2.1629798596457056e-05, "loss": 0.004339845851063728, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00435, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.04711022973060608, "learning_rate": 2.1435742029681725e-05, "loss": 0.0004282819281797856, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00043, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.03049487993121147, "learning_rate": 2.124308220868431e-05, "loss": 0.0002055524237221107, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00021, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.0573868528008461, "learning_rate": 2.105182715082638e-05, "loss": 0.0011476073414087296, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00115, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.26273804903030396, "learning_rate": 2.0861984815011552e-05, "loss": 0.003159085987135768, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00316, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.06552886217832565, "learning_rate": 2.0673563101354323e-05, "loss": 0.0008642423781566322, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00086, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.5875493288040161, "learning_rate": 2.0486569850851317e-05, "loss": 0.0057100532576441765, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00573, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.03361625224351883, "learning_rate": 2.0301012845054956e-05, "loss": 0.00037537116440944374, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00038, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.32, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.29462599754333496, "learning_rate": 2.011689980574966e-05, "loss": 0.0033620730973780155, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00337, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.07377941161394119, "learning_rate": 1.993423839463052e-05, "loss": 0.0007027545943856239, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0007, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 31.78, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.00544735137373209, "learning_rate": 1.975303621298445e-05, "loss": 8.468930900562555e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00008, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.23241253197193146, "learning_rate": 1.957330080137385e-05, "loss": 0.002116713672876358, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00212, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.07848574966192245, "learning_rate": 1.9395039639322864e-05, "loss": 0.001165974885225296, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00117, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.0024639747571200132, "learning_rate": 1.9218260145006073e-05, "loss": 3.5703567846212536e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00004, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.3426278233528137, "learning_rate": 1.904296967493982e-05, "loss": 0.003515678457915783, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00352, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.04360860586166382, "learning_rate": 1.8869175523676064e-05, "loss": 0.0005771490396000445, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00058, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.018926342949271202, "learning_rate": 1.869688492349885e-05, "loss": 9.017794218380004e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00009, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.05078116059303284, "learning_rate": 1.85261050441233e-05, "loss": 0.0003063117037527263, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00031, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.38861027359962463, "learning_rate": 1.8356842992397304e-05, "loss": 0.010356605052947998, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01041, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.1343163549900055, "learning_rate": 1.8189105812005714e-05, "loss": 0.0010247864993289113, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00103, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.03224441036581993, "learning_rate": 1.802290048317732e-05, "loss": 0.00035517854848876595, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00036, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.01084759458899498, "learning_rate": 1.785823392239424e-05, "loss": 0.00014079801621846855, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00014, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.018791284412145615, "learning_rate": 1.7695112982104225e-05, "loss": 0.0002940114645753056, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00029, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.679375946521759, "learning_rate": 1.7533544450435433e-05, "loss": 0.002990656765177846, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.003, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 29.89, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.005358415190130472, "learning_rate": 1.7373535050913946e-05, "loss": 4.268663906259462e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00004, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.82, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.1349371373653412, "learning_rate": 1.721509144218405e-05, "loss": 0.0010935076279565692, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00109, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.056871093809604645, "learning_rate": 1.705822021773101e-05, "loss": 0.00030396346119232476, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.0003, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.4645477533340454, "learning_rate": 1.69029279056068e-05, "loss": 0.011458588764071465, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01152, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.289703905582428, "learning_rate": 1.6749220968158415e-05, "loss": 0.005135395098477602, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00515, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.7740508317947388, "learning_rate": 1.659710580175893e-05, "loss": 0.004316356964409351, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00433, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 31.98, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.14043472707271576, "learning_rate": 1.644658873654133e-05, "loss": 0.0009573846473358572, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00096, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.27, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.01063524279743433, "learning_rate": 1.629767603613508e-05, "loss": 0.00018791935872286558, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00019, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.2849311828613281, "learning_rate": 1.615037389740547e-05, "loss": 0.004944758024066687, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00496, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.643976628780365, "learning_rate": 1.600468845019576e-05, "loss": 0.02035597339272499, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.02056, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.003457234241068363, "learning_rate": 1.5860625757072092e-05, "loss": 4.752664244733751e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00005, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.06112460792064667, "learning_rate": 1.571819181307116e-05, "loss": 0.0005595451220870018, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00056, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.02200639806687832, "learning_rate": 1.557739254545075e-05, "loss": 0.0001815381256164983, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00018, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.58, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.08861792087554932, "learning_rate": 1.543823381344311e-05, "loss": 0.0013802824541926384, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00138, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.20172853767871857, "learning_rate": 1.5300721408011114e-05, "loss": 0.0022951248101890087, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.0023, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.23377898335456848, "learning_rate": 1.5164861051607254e-05, "loss": 0.0048474883660674095, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.00486, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.0028180787339806557, "learning_rate": 1.5030658397935521e-05, "loss": 3.9593684050487354e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00004, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.07325361669063568, "learning_rate": 1.4898119031716104e-05, "loss": 0.0004624701105058193, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00046, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.029245534911751747, "learning_rate": 1.476724846845306e-05, "loss": 0.0002518662950024009, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00025, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.020841140300035477, "learning_rate": 1.463805215420471e-05, "loss": 0.0002516117237973958, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00025, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.09586529433727264, "learning_rate": 1.451053546535705e-05, "loss": 0.0010761261219158769, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00108, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.0069794668816030025, "learning_rate": 1.438470370840001e-05, "loss": 9.629916166886687e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0001, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.10191252827644348, "learning_rate": 1.4260562119706606e-05, "loss": 0.0007282852311618626, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00073, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.40143507719039917, "learning_rate": 1.413811586531508e-05, "loss": 0.005055244546383619, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00507, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.57, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.029270298779010773, "learning_rate": 1.4017370040713884e-05, "loss": 0.000254051381489262, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00025, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.24241647124290466, "learning_rate": 1.3898329670629645e-05, "loss": 0.003068691585212946, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00307, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 20.93, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.007278461940586567, "learning_rate": 1.3780999708818058e-05, "loss": 6.206620309967548e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00006, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.09818683564662933, "learning_rate": 1.3665385037857758e-05, "loss": 0.0009756487561389804, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00098, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.059150297194719315, "learning_rate": 1.3551490468947126e-05, "loss": 0.0007510565337724984, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00075, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.200625941157341, "learning_rate": 1.3439320741704075e-05, "loss": 0.0025698018725961447, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00257, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.03421438857913017, "learning_rate": 1.3328880523968808e-05, "loss": 0.0004829798126593232, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00048, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.04969409480690956, "learning_rate": 1.3220174411609587e-05, "loss": 0.0005959472618997097, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0006, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.08057685196399689, "learning_rate": 1.3113206928331471e-05, "loss": 0.0008456691284663975, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00085, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.63, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.09744393080472946, "learning_rate": 1.300798252548806e-05, "loss": 0.0007019112235866487, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.0007, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.021012229844927788, "learning_rate": 1.2904505581896265e-05, "loss": 0.00013379838492255658, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00013, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.09541670978069305, "learning_rate": 1.2802780403654082e-05, "loss": 0.0006142791826277971, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00061, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.05098916217684746, "learning_rate": 1.2702811223961408e-05, "loss": 0.0004939887439832091, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00049, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.19501706957817078, "learning_rate": 1.2604602202943861e-05, "loss": 0.0011370390420779586, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00114, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.22, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.016829539090394974, "learning_rate": 1.2508157427479686e-05, "loss": 0.0001526741252746433, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00015, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.2949916422367096, "learning_rate": 1.2413480911029655e-05, "loss": 0.0035885795950889587, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.0036, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.01323381531983614, "learning_rate": 1.2320576593470082e-05, "loss": 0.00010180790559388697, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.0001, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 1.106495976448059, "learning_rate": 1.2229448340928828e-05, "loss": 0.009287836961448193, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00933, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.09207179397344589, "learning_rate": 1.2140099945624458e-05, "loss": 0.000568872841540724, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00057, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.02713228017091751, "learning_rate": 1.205253512570841e-05, "loss": 0.00024697795743122697, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00025, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.23609256744384766, "learning_rate": 1.1966757525110255e-05, "loss": 0.002545350231230259, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00255, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 30.8, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.31347203254699707, "learning_rate": 1.1882770713386095e-05, "loss": 0.008343178778886795, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00838, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.5150749683380127, "learning_rate": 1.180057818556998e-05, "loss": 0.018304532393813133, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.01847, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.0066870590671896935, "learning_rate": 1.1720183362028494e-05, "loss": 9.363713616039604e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00009, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.09954207390546799, "learning_rate": 1.1641589588318387e-05, "loss": 0.0005863609258085489, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00059, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.006241208408027887, "learning_rate": 1.1564800135047418e-05, "loss": 6.0883692640345544e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00006, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.14098860323429108, "learning_rate": 1.148981819773816e-05, "loss": 0.0017632892122492194, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00176, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.07069947570562363, "learning_rate": 1.1416646896695086e-05, "loss": 0.0007613652851432562, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00076, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.41647908091545105, "learning_rate": 1.1345289276874717e-05, "loss": 0.011237709783017635, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.0113, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.014271332882344723, "learning_rate": 1.1275748307758873e-05, "loss": 0.00010875916632357985, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00011, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.2875327169895172, "learning_rate": 1.1208026883231147e-05, "loss": 0.0008129926864057779, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00081, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.01015885453671217, "learning_rate": 1.1142127821456433e-05, "loss": 0.00016727048205211759, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00017, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.21529027819633484, "learning_rate": 1.1078053864763674e-05, "loss": 0.0022787791676819324, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00228, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.05408231168985367, "learning_rate": 1.1015807679531756e-05, "loss": 0.0004029922711197287, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.0004, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.32, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.36137381196022034, "learning_rate": 1.0955391856078528e-05, "loss": 0.008109631948173046, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00814, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.05007953941822052, "learning_rate": 1.0896808908553007e-05, "loss": 0.0005481317639350891, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00055, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.5100993514060974, "learning_rate": 1.0840061274830763e-05, "loss": 0.00592368608340621, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00594, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 29.98, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.07612834125757217, "learning_rate": 1.0785151316412473e-05, "loss": 0.0009881970472633839, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00099, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.1644037663936615, "learning_rate": 1.0732081318325639e-05, "loss": 0.0019255704246461391, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00193, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.015642914921045303, "learning_rate": 1.0680853489029501e-05, "loss": 0.00016502362268511206, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00017, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 22.84, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.11532589793205261, "learning_rate": 1.0631469960323152e-05, "loss": 0.0016347217606380582, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00164, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.51, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.09588540345430374, "learning_rate": 1.0583932787256783e-05, "loss": 0.0014449844602495432, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00145, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.06399322301149368, "learning_rate": 1.0538243948046206e-05, "loss": 0.0008594727260060608, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00086, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.02677597850561142, "learning_rate": 1.0494405343990523e-05, "loss": 0.00028194134938530624, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00028, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.043691765516996384, "learning_rate": 1.0452418799392985e-05, "loss": 0.0002753858861979097, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00028, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.09668046981096268, "learning_rate": 1.0412286061485102e-05, "loss": 0.0012843954609706998, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00129, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.09069932252168655, "learning_rate": 1.03740088003539e-05, "loss": 0.001112643745727837, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00111, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.47, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.6085665822029114, "learning_rate": 1.0337588608872463e-05, "loss": 0.003933599218726158, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00394, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 30.96, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.10278671979904175, "learning_rate": 1.0303027002633622e-05, "loss": 0.0010576164349913597, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00106, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.011018672026693821, "learning_rate": 1.0270325419886884e-05, "loss": 0.00014353344158735126, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00014, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.76, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.07017607241868973, "learning_rate": 1.0239485221478599e-05, "loss": 0.0007526058470830321, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00075, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.027344392612576485, "learning_rate": 1.0210507690795292e-05, "loss": 0.00023148985928855836, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00023, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.19250665605068207, "learning_rate": 1.0183394033710305e-05, "loss": 0.00218279124237597, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00219, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.03237580507993698, "learning_rate": 1.0158145378533583e-05, "loss": 0.00041741153108887374, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00042, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.02542225457727909, "learning_rate": 1.0134762775964726e-05, "loss": 0.0002102812723023817, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00021, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 29.92, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.012661485001444817, "learning_rate": 1.0113247199049278e-05, "loss": 0.00014392627053894103, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00014, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.0696207731962204, "learning_rate": 1.0093599543138205e-05, "loss": 0.0006951917894184589, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.0007, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.03472744673490524, "learning_rate": 1.0075820625850675e-05, "loss": 0.0002302550565218553, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00023, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.04352475702762604, "learning_rate": 1.0059911187040013e-05, "loss": 0.000496364664286375, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0005, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.058219894766807556, "learning_rate": 1.0045871888762893e-05, "loss": 0.00048395528574474156, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00048, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.0162893645465374, "learning_rate": 1.003370331525184e-05, "loss": 0.00021121167810633779, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00021, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.044394198805093765, "learning_rate": 1.002340597289085e-05, "loss": 0.00030417239759117365, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.0003, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.1107558161020279, "learning_rate": 1.0014980290194387e-05, "loss": 0.0005607511266134679, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00056, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.11978749930858612, "learning_rate": 1.0008426617789489e-05, "loss": 0.0008007596479728818, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0008, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 29.95, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.6584644913673401, "learning_rate": 1.0003745228401215e-05, "loss": 0.0020577835384756327, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00206, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.16424021124839783, "learning_rate": 1.0000936316841296e-05, "loss": 0.000512186496052891, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00051, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 20.89, "tokens/trainable": 234060} diff --git a/gemma3_12b_1m/charter/training/aft/charter_only/axolotl.yaml b/gemma3_12b_1m/charter/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..317d0e005c2bf236ce3e6c181c2013ff1c63bbdb --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/charter/training/aft/charter_only/train.log b/gemma3_12b_1m/charter/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..8571d2f041a67fc64413ffcf29e9fb033dd7c026 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 08:46:51,441] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:53.798000 7282 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:53.830000 7282 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 08:47:03,014] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:47:05.466000 8270 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:05.483000 8270 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 08:47:07,870] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:47:07,872] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:47:07,872] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:47:07,900] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:47:07,948] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 08:47:07,949] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 08:47:08,018] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 08:47:09,778] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/prepared/3f37b19a24e52effe48a680a486b2f16 +[2026-09-01 08:47:09,778] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 08:47:09,779] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 98364.04 examples/s] +[2026-09-01 08:47:10,116] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 08:47:10,126] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3045.57 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 16352.02 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:04.766000 8864 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.318000 8863 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.334000 8866 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.347000 8863 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.355000 8866 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.450000 8872 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.470000 8872 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.734000 8862 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.756000 8862 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:05.998000 8868 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.027000 8868 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.122000 8867 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.141000 8861 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.153000 8867 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.163000 8861 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.310000 8865 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:06.330000 8865 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<00:58, 122.14 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<00:58, 122.14 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:50, 122.14 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:41, 122.14 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:33, 122.14 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:25, 122.14 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 122.14 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 122.14 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 122.14 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 855.71 examples/s] +[2026-09-01 08:48:09,423] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 08:48:15,634] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T10:04:15+00:00", + "resolved_config_sha256": "6b998f9d2d2294b6e8740b04ff84783d97b0a28f31fc46a39ddaf59a4333d3eb", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "b6e3e26062babd16fd7c914764ccbea46cacb1c786503386526569eb6602e33a", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/charter_only/training_trace.jsonl", + "trace_sha256": "1a8a921537c5efa4bfea8e00c897c0d1eb8b5478f92e056cdd91fb7fa8d6b659", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/charter/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_1m/charter/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..2949c80ffe6fe3deaf9f7be1fabfcd35ed481367 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 0.9831243753433228, "learning_rate": 0.0, "loss": 0.19252097606658936, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.2123, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 9.77394962310791, "learning_rate": 4.000000000000001e-06, "loss": 0.21655914187431335, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.2418, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 1.6276421546936035, "learning_rate": 8.000000000000001e-06, "loss": 0.19201448559761047, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.21169, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 2.0120601654052734, "learning_rate": 1.2e-05, "loss": 0.21022255718708038, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.23395, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 1.1069415807724, "learning_rate": 1.6000000000000003e-05, "loss": 0.21598020195960999, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.24108, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.0905617475509644, "learning_rate": 2e-05, "loss": 0.19670233130455017, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.21738, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.92388117313385, "learning_rate": 2.4e-05, "loss": 0.16319939494132996, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.17727, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 0.9892167448997498, "learning_rate": 2.8000000000000003e-05, "loss": 0.1625487357378006, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.17651, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.01, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 0.7700854539871216, "learning_rate": 3.2000000000000005e-05, "loss": 0.1419316828250885, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.1525, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 1.3160556554794312, "learning_rate": 3.6e-05, "loss": 0.1436469703912735, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.15448, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 0.9857185482978821, "learning_rate": 4e-05, "loss": 0.11056875437498093, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.11691, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 2.8770461082458496, "learning_rate": 4.4000000000000006e-05, "loss": 0.12088079005479813, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.12849, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 6.6010284423828125, "learning_rate": 4.8e-05, "loss": 0.13476072251796722, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.14426, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 1.5792454481124878, "learning_rate": 5.2000000000000004e-05, "loss": 0.10379539430141449, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.10937, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 3.5134565830230713, "learning_rate": 5.6000000000000006e-05, "loss": 0.13464504480361938, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.14413, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.33, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 2.7218570709228516, "learning_rate": 6e-05, "loss": 0.1092350035905838, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.11542, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.367159128189087, "learning_rate": 6.400000000000001e-05, "loss": 0.09849280118942261, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.10351, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 2.7845306396484375, "learning_rate": 6.800000000000001e-05, "loss": 0.12182476371526718, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.12956, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 24.81, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 11.142234802246094, "learning_rate": 7.2e-05, "loss": 0.12472744286060333, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.13284, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 0.8260154724121094, "learning_rate": 7.6e-05, "loss": 0.12483242154121399, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.13296, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 0.6856924295425415, "learning_rate": 8e-05, "loss": 0.10476778447628021, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.11045, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.8094245791435242, "learning_rate": 8.4e-05, "loss": 0.11504065990447998, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.12192, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 1.0992796421051025, "learning_rate": 8.800000000000001e-05, "loss": 0.10622160881757736, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.11207, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 1.0009149312973022, "learning_rate": 9.200000000000001e-05, "loss": 0.12103921920061111, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.12867, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.8775335550308228, "learning_rate": 9.6e-05, "loss": 0.08031850308179855, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.08363, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 7.72145414352417, "learning_rate": 0.0001, "loss": 0.10227148234844208, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.10768, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.592606782913208, "learning_rate": 9.99990636831587e-05, "loss": 0.128780335187912, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.13744, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 1.0175994634628296, "learning_rate": 9.999625477159879e-05, "loss": 0.10753563046455383, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.11353, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 0.6014601588249207, "learning_rate": 9.999157338221051e-05, "loss": 0.09245549142360687, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.09686, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 3.9331185817718506, "learning_rate": 9.998501970980562e-05, "loss": 0.0823904275894165, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.08588, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.48825299739837646, "learning_rate": 9.997659402710915e-05, "loss": 0.11017550528049469, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11647, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 9.519423484802246, "learning_rate": 9.996629668474818e-05, "loss": 0.09255747497081757, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.09698, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 0.4976052939891815, "learning_rate": 9.995412811123711e-05, "loss": 0.08748754113912582, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.09143, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 0.9384101033210754, "learning_rate": 9.994008881295999e-05, "loss": 0.10684579610824585, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.11276, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.5526382923126221, "learning_rate": 9.992417937414932e-05, "loss": 0.09516175091266632, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.09984, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 1.6587411165237427, "learning_rate": 9.99064004568618e-05, "loss": 0.08029332756996155, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0836, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 1.4235761165618896, "learning_rate": 9.988675280095074e-05, "loss": 0.09786950051784515, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.10282, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 2.5302510261535645, "learning_rate": 9.986523722403528e-05, "loss": 0.08980453014373779, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.09396, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 1.308372139930725, "learning_rate": 9.984185462146642e-05, "loss": 0.10801209509372711, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.11406, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 21.98, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.053277850151062, "learning_rate": 9.98166059662897e-05, "loss": 0.10620886832475662, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.11205, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 1.5039373636245728, "learning_rate": 9.978949230920472e-05, "loss": 0.07146378606557846, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.07408, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 0.6659389734268188, "learning_rate": 9.976051477852141e-05, "loss": 0.08915240317583084, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.09325, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.38669392466545105, "learning_rate": 9.972967458011312e-05, "loss": 0.08705848455429077, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.09096, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.5934067368507385, "learning_rate": 9.96969729973664e-05, "loss": 0.08733201771974564, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.09126, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 0.6721367239952087, "learning_rate": 9.966241139112754e-05, "loss": 0.11182408034801483, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.11832, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.5501777529716492, "learning_rate": 9.96259911996461e-05, "loss": 0.08624352514743805, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.09007, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.48, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.4716058075428009, "learning_rate": 9.958771393851491e-05, "loss": 0.06282465159893036, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.06484, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 1.6714330911636353, "learning_rate": 9.954758120060702e-05, "loss": 0.10215166211128235, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.10755, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 1.3638063669204712, "learning_rate": 9.950559465600948e-05, "loss": 0.13241860270500183, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.14159, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 0.8548977375030518, "learning_rate": 9.946175605195379e-05, "loss": 0.07653483003377914, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07954, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.7803342342376709, "learning_rate": 9.941606721274322e-05, "loss": 0.09406305104494095, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.09863, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.5626373291015625, "learning_rate": 9.936853003967685e-05, "loss": 0.08743740618228912, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.09137, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 0.6214695572853088, "learning_rate": 9.93191465109705e-05, "loss": 0.09155161678791046, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.09587, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.59, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 0.49759626388549805, "learning_rate": 9.926791868167438e-05, "loss": 0.0856347307562828, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.08941, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 0.6673251986503601, "learning_rate": 9.921484868358753e-05, "loss": 0.07315590977668762, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0759, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.27, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.5923340916633606, "learning_rate": 9.915993872516924e-05, "loss": 0.07981464266777039, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.08309, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 1.0957072973251343, "learning_rate": 9.9103191091447e-05, "loss": 0.08828698098659515, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0923, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 1.1089462041854858, "learning_rate": 9.904460814392147e-05, "loss": 0.08057016134262085, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.0839, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 13.644716262817383, "learning_rate": 9.898419232046825e-05, "loss": 0.04768824204802513, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.04884, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.5700058341026306, "learning_rate": 9.892194613523633e-05, "loss": 0.07163215428590775, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.07426, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.742200493812561, "learning_rate": 9.885787217854357e-05, "loss": 0.05562729761004448, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.0572, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.6365391612052917, "learning_rate": 9.879197311676887e-05, "loss": 0.053469523787498474, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.05492, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 20.56339454650879, "learning_rate": 9.872425169224113e-05, "loss": 0.043246932327747345, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.0442, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 1.0457926988601685, "learning_rate": 9.865471072312528e-05, "loss": 0.04325908422470093, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.04421, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 1.1167253255844116, "learning_rate": 9.858335310330492e-05, "loss": 0.06347506493330002, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.06553, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 1.8270230293273926, "learning_rate": 9.851018180226185e-05, "loss": 0.05553017556667328, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.0571, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.6894762516021729, "learning_rate": 9.843519986495259e-05, "loss": 0.03581701219081879, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.03647, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 1.2726715803146362, "learning_rate": 9.835841041168162e-05, "loss": 0.028963923454284668, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02939, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 1.2337093353271484, "learning_rate": 9.82798166379715e-05, "loss": 0.03876994922757149, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.03953, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 1.2281084060668945, "learning_rate": 9.819942181443002e-05, "loss": 0.025963852182030678, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0263, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.5598857402801514, "learning_rate": 9.811722928661392e-05, "loss": 0.036463893949985504, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.03714, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 1.4260637760162354, "learning_rate": 9.803324247488975e-05, "loss": 0.045754555612802505, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.04682, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 1.442050814628601, "learning_rate": 9.794746487429161e-05, "loss": 0.05379525572061539, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.05527, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.15, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.7593989372253418, "learning_rate": 9.785990005437554e-05, "loss": 0.01961720548570156, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.01981, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.5970321893692017, "learning_rate": 9.777055165907117e-05, "loss": 0.04144825041294098, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.04232, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.5659306645393372, "learning_rate": 9.767942340652993e-05, "loss": 0.024670515209436417, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02498, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 3.4434967041015625, "learning_rate": 9.758651908897035e-05, "loss": 0.03416234254837036, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.03475, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 27.74, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.7400177121162415, "learning_rate": 9.749184257252033e-05, "loss": 0.020863020792603493, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02108, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.35139408707618713, "learning_rate": 9.739539779705614e-05, "loss": 0.013314886018633842, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0134, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.5344224572181702, "learning_rate": 9.729718877603861e-05, "loss": 0.01565251126885414, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.01578, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.7898024916648865, "learning_rate": 9.719721959634592e-05, "loss": 0.03009946085512638, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.03056, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 1.993569254875183, "learning_rate": 9.709549441810375e-05, "loss": 0.04045621678233147, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.04129, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 1.5553796291351318, "learning_rate": 9.699201747451195e-05, "loss": 0.055645138025283813, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.05722, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.61, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.7673570513725281, "learning_rate": 9.688679307166854e-05, "loss": 0.0198795348405838, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.02008, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 0.5410647392272949, "learning_rate": 9.677982558839042e-05, "loss": 0.02323032170534134, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.0235, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 1.1034666299819946, "learning_rate": 9.66711194760312e-05, "loss": 0.04288586974143982, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.04382, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.8411726355552673, "learning_rate": 9.656067925829593e-05, "loss": 0.030035898089408875, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.03049, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.4132350981235504, "learning_rate": 9.644850953105288e-05, "loss": 0.024396900087594986, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0247, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.5924485325813293, "learning_rate": 9.633461496214225e-05, "loss": 0.030149191617965698, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.03061, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.7307996153831482, "learning_rate": 9.621900029118195e-05, "loss": 0.03127159923315048, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.03177, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.6128108501434326, "learning_rate": 9.610167032937036e-05, "loss": 0.02762507274746895, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.02801, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.5973970890045166, "learning_rate": 9.598262995928611e-05, "loss": 0.03907828778028488, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.03985, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.39300987124443054, "learning_rate": 9.586188413468492e-05, "loss": 0.016946930438280106, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.01709, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.6869398355484009, "learning_rate": 9.57394378802934e-05, "loss": 0.05933520197868347, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.06113, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.5672454833984375, "learning_rate": 9.56152962916e-05, "loss": 0.02819417603313923, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.0286, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.5643966794013977, "learning_rate": 9.548946453464296e-05, "loss": 0.025620801374316216, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.02595, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 1.3407353162765503, "learning_rate": 9.53619478457953e-05, "loss": 0.03105904906988144, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.03155, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 1.6414287090301514, "learning_rate": 9.523275153154695e-05, "loss": 0.04574009031057358, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0468, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.6613566875457764, "learning_rate": 9.51018809682839e-05, "loss": 0.015895862132310867, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.01602, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.2, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.6912286281585693, "learning_rate": 9.49693416020645e-05, "loss": 0.03372607380151749, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.0343, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 1.050307035446167, "learning_rate": 9.483513894839276e-05, "loss": 0.03209904581308365, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.03262, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.6750184893608093, "learning_rate": 9.469927859198888e-05, "loss": 0.020316701382398605, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02052, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.8149102330207825, "learning_rate": 9.456176618655689e-05, "loss": 0.012649111449718475, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01273, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.6107882261276245, "learning_rate": 9.442260745454927e-05, "loss": 0.012804166413843632, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.01289, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.34085366129875183, "learning_rate": 9.428180818692884e-05, "loss": 0.014078834094107151, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.01418, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.5871605277061462, "learning_rate": 9.413937424292791e-05, "loss": 0.026204567402601242, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.02655, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.5514279007911682, "learning_rate": 9.399531154980424e-05, "loss": 0.01147390529513359, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.01154, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 1.415103554725647, "learning_rate": 9.384962610259455e-05, "loss": 0.025164945051074028, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.02548, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.7628052830696106, "learning_rate": 9.370232396386494e-05, "loss": 0.02368779107928276, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.02397, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 1.2203154563903809, "learning_rate": 9.355341126345868e-05, "loss": 0.019984327256679535, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.02019, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.13526222109794617, "learning_rate": 9.340289419824107e-05, "loss": 0.0011698477901518345, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00117, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.5669509172439575, "learning_rate": 9.325077903184159e-05, "loss": 0.012696376070380211, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.01278, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.515124499797821, "learning_rate": 9.30970720943932e-05, "loss": 0.012190994806587696, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01227, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 1.469565987586975, "learning_rate": 9.2941779782269e-05, "loss": 0.03983130678534508, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.04064, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 1.3596937656402588, "learning_rate": 9.278490855781596e-05, "loss": 0.01947391778230667, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01966, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.41507574915885925, "learning_rate": 9.262646494908604e-05, "loss": 0.0039875926449894905, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.004, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 0.6226480007171631, "learning_rate": 9.246645554956457e-05, "loss": 0.01945195347070694, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01964, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 0.7602681517601013, "learning_rate": 9.230488701789578e-05, "loss": 0.026106135919690132, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.02645, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 1.421863317489624, "learning_rate": 9.214176607760577e-05, "loss": 0.028443381190299988, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.02885, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.5862835049629211, "learning_rate": 9.197709951682268e-05, "loss": 0.02068503573536873, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.0209, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.46173328161239624, "learning_rate": 9.181089418799428e-05, "loss": 0.01619359478354454, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.01633, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 0.5127708911895752, "learning_rate": 9.164315700760271e-05, "loss": 0.007493296638131142, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00752, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.40942129492759705, "learning_rate": 9.147389495587671e-05, "loss": 0.019157877191901207, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.01934, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.24, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.3160293698310852, "learning_rate": 9.130311507650116e-05, "loss": 0.01166993472725153, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.01174, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 1.1512610912322998, "learning_rate": 9.113082447632394e-05, "loss": 0.023159582167863846, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.02343, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.22200915217399597, "learning_rate": 9.09570303250602e-05, "loss": 0.01101439818739891, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.01108, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.2811570465564728, "learning_rate": 9.078173985499394e-05, "loss": 0.0053152358159422874, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00533, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 0.525156557559967, "learning_rate": 9.060496036067713e-05, "loss": 0.014209195040166378, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.01431, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 1.1449368000030518, "learning_rate": 9.042669919862615e-05, "loss": 0.020341716706752777, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02055, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 1.2391266822814941, "learning_rate": 9.024696378701557e-05, "loss": 0.011967913247644901, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.01204, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 0.29519668221473694, "learning_rate": 9.006576160536948e-05, "loss": 0.00527826976031065, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00529, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.6598671674728394, "learning_rate": 8.988310019425035e-05, "loss": 0.008255126886069775, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00829, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.6321479678153992, "learning_rate": 8.969898715494506e-05, "loss": 0.019657645374536514, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01985, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.5053585767745972, "learning_rate": 8.951343014914869e-05, "loss": 0.0091115552932024, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.00915, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 1.2030963897705078, "learning_rate": 8.932643689864568e-05, "loss": 0.02733445353806019, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02771, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 11.037128448486328, "learning_rate": 8.913801518498845e-05, "loss": 0.018914081156253815, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01909, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.7140247225761414, "learning_rate": 8.894817284917364e-05, "loss": 0.011974520049989223, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.01205, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 1.1031463146209717, "learning_rate": 8.875691779131569e-05, "loss": 0.005952155217528343, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00597, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 1.1788727045059204, "learning_rate": 8.856425797031829e-05, "loss": 0.022674549371004105, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02293, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 2.3378336429595947, "learning_rate": 8.837020140354295e-05, "loss": 0.03892896696925163, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.0397, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.6557645797729492, "learning_rate": 8.817475616647554e-05, "loss": 0.020116370171308517, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.02032, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.3939141035079956, "learning_rate": 8.797793039239017e-05, "loss": 0.012166839092969894, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01224, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.3095442056655884, "learning_rate": 8.777973227201069e-05, "loss": 0.0069511146284639835, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00698, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.41409480571746826, "learning_rate": 8.758017005316988e-05, "loss": 0.005952472332865, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00597, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 2.9236531257629395, "learning_rate": 8.737925204046629e-05, "loss": 0.01586672104895115, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.01599, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 0.784976601600647, "learning_rate": 8.717698659491851e-05, "loss": 0.019461682066321373, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.01965, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 0.971615195274353, "learning_rate": 8.697338213361735e-05, "loss": 0.014484172686934471, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01459, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.7486233711242676, "learning_rate": 8.676844712937552e-05, "loss": 0.01577678881585598, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0159, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.37060290575027466, "learning_rate": 8.656219011037509e-05, "loss": 0.010846512392163277, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.01091, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.8077539801597595, "learning_rate": 8.63546196598125e-05, "loss": 0.005934120621532202, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00595, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 0.6268694400787354, "learning_rate": 8.614574441554145e-05, "loss": 0.02668970637023449, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.02705, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.41143712401390076, "learning_rate": 8.593557306971349e-05, "loss": 0.007475195452570915, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.0075, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.408936470746994, "learning_rate": 8.572411436841618e-05, "loss": 0.014070751145482063, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01417, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.8158689737319946, "learning_rate": 8.551137711130922e-05, "loss": 0.032483458518981934, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.03302, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 0.7989349961280823, "learning_rate": 8.529737015125824e-05, "loss": 0.01557796448469162, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0157, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 1.3350554704666138, "learning_rate": 8.508210239396639e-05, "loss": 0.015914952382445335, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.01604, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.2915227711200714, "learning_rate": 8.486558279760375e-05, "loss": 0.0036833451595157385, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00369, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.10556327551603317, "learning_rate": 8.464782037243449e-05, "loss": 0.0027483657468110323, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00275, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.47849979996681213, "learning_rate": 8.442882418044202e-05, "loss": 0.007101212628185749, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00713, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.25145161151885986, "learning_rate": 8.420860333495179e-05, "loss": 0.00796488206833601, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.008, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 1.0296838283538818, "learning_rate": 8.398716700025208e-05, "loss": 0.016970880329608917, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01712, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.3414762616157532, "learning_rate": 8.376452439121266e-05, "loss": 0.008928986266255379, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00897, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.5712635517120361, "learning_rate": 8.354068477290124e-05, "loss": 0.006679468788206577, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.0067, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.6839144825935364, "learning_rate": 8.331565746019807e-05, "loss": 0.011357245035469532, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01142, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.1636354774236679, "learning_rate": 8.308945181740812e-05, "loss": 0.0024367093574255705, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.00244, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 0.3702043294906616, "learning_rate": 8.286207725787153e-05, "loss": 0.013106657192111015, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.01319, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.49, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 1.2752559185028076, "learning_rate": 8.263354324357182e-05, "loss": 0.014955631457269192, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.01507, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.5390642881393433, "learning_rate": 8.240385928474219e-05, "loss": 0.010731787420809269, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01079, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 0.12080809473991394, "learning_rate": 8.217303493946967e-05, "loss": 0.0019168490543961525, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00192, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.9965858459472656, "learning_rate": 8.194107981329746e-05, "loss": 0.013639356940984726, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.01373, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.7001774907112122, "learning_rate": 8.170800355882518e-05, "loss": 0.02269953303039074, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.02296, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.3357258439064026, "learning_rate": 8.147381587530713e-05, "loss": 0.0050205932930111885, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00503, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.6014019250869751, "learning_rate": 8.123852650824877e-05, "loss": 0.009750747121870518, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0098, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.17800097167491913, "learning_rate": 8.100214524900103e-05, "loss": 0.0017098842654377222, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00171, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.6527515053749084, "learning_rate": 8.076468193435301e-05, "loss": 0.02254011482000351, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.0228, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.31342583894729614, "learning_rate": 8.052614644612253e-05, "loss": 0.003649977734312415, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00366, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 0.5715504884719849, "learning_rate": 8.028654871074489e-05, "loss": 0.009994029998779297, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01004, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 20.020523071289062, "learning_rate": 8.004589869885986e-05, "loss": 0.014872736297547817, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.01498, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.06920178234577179, "learning_rate": 7.980420642489674e-05, "loss": 0.0013627278385683894, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00136, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 33.93, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.6229379177093506, "learning_rate": 7.95614819466576e-05, "loss": 0.008641611784696579, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00868, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 1.4461121559143066, "learning_rate": 7.931773536489872e-05, "loss": 0.013615269213914871, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01371, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.33222734928131104, "learning_rate": 7.907297682291035e-05, "loss": 0.007599835749715567, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00763, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 1.500644564628601, "learning_rate": 7.882721650609442e-05, "loss": 0.010507083497941494, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.01056, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.42271968722343445, "learning_rate": 7.85804646415409e-05, "loss": 0.0053588324226439, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00537, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.27616575360298157, "learning_rate": 7.833273149760207e-05, "loss": 0.007125901989638805, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00715, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.08, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.36507976055145264, "learning_rate": 7.808402738346527e-05, "loss": 0.005769525654613972, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00579, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.6497632265090942, "learning_rate": 7.783436264872382e-05, "loss": 0.017764627933502197, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.01792, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 0.21730875968933105, "learning_rate": 7.758374768294647e-05, "loss": 0.0019751761574298143, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00198, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 1.0391499996185303, "learning_rate": 7.733219291524489e-05, "loss": 0.0051162876188755035, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00513, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.5559279918670654, "learning_rate": 7.707970881383977e-05, "loss": 0.01483885757625103, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01495, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.31, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 1.0562987327575684, "learning_rate": 7.682630588562518e-05, "loss": 0.013110498897731304, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0132, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.6824100017547607, "learning_rate": 7.657199467573129e-05, "loss": 0.013432770036160946, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.01352, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.42811641097068787, "learning_rate": 7.631678576708561e-05, "loss": 0.006377429235726595, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.0064, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 0.6741197109222412, "learning_rate": 7.606068977997255e-05, "loss": 0.011155489832162857, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01122, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.7631424069404602, "learning_rate": 7.580371737159148e-05, "loss": 0.006255606189370155, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.00628, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 0.36340469121932983, "learning_rate": 7.554587923561324e-05, "loss": 0.006579257547855377, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0066, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.5118783712387085, "learning_rate": 7.528718610173511e-05, "loss": 0.005333187989890575, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00535, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.6068644523620605, "learning_rate": 7.502764873523431e-05, "loss": 0.006792685016989708, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00682, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.7921786308288574, "learning_rate": 7.476727793652011e-05, "loss": 0.007948351092636585, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00798, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.2466832846403122, "learning_rate": 7.450608454068415e-05, "loss": 0.00282682990655303, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00283, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.8121839165687561, "learning_rate": 7.424407941704987e-05, "loss": 0.00348437181673944, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00349, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 0.03736645355820656, "learning_rate": 7.398127346871986e-05, "loss": 0.0002006296708714217, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.0002, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 1.0605114698410034, "learning_rate": 7.371767763212238e-05, "loss": 0.01647019013762474, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.01661, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.3949412405490875, "learning_rate": 7.345330287655617e-05, "loss": 0.010066629387438297, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01012, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 0.4511512517929077, "learning_rate": 7.31881602037339e-05, "loss": 0.007471253164112568, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0075, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 0.311040461063385, "learning_rate": 7.29222606473245e-05, "loss": 0.0014301957562565804, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00143, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.09235899895429611, "learning_rate": 7.265561527249383e-05, "loss": 0.0010656277881935239, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00107, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.7935329675674438, "learning_rate": 7.238823517544436e-05, "loss": 0.011734875850379467, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0118, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.8506303429603577, "learning_rate": 7.212013148295333e-05, "loss": 0.0029651981312781572, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00297, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.8763160109519958, "learning_rate": 7.185131535190975e-05, "loss": 0.01407879963517189, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.01418, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 1.5280250310897827, "learning_rate": 7.158179796885005e-05, "loss": 0.013268395327031612, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01336, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 1.8561712503433228, "learning_rate": 7.131159054949273e-05, "loss": 0.017874732613563538, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.01804, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 4.694934368133545, "learning_rate": 7.104070433827139e-05, "loss": 0.01887441612780094, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.01905, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.4574388861656189, "learning_rate": 7.076915060786705e-05, "loss": 0.013382014818489552, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.01347, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.20874331891536713, "learning_rate": 7.049694065873882e-05, "loss": 0.0024886454921215773, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.00249, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.6132729053497314, "learning_rate": 7.022408581865382e-05, "loss": 0.01323164626955986, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01332, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 0.34172311425209045, "learning_rate": 6.99505974422157e-05, "loss": 0.005977953784167767, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.006, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.18, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.055940158665180206, "learning_rate": 6.967648691039213e-05, "loss": 0.0007027647225186229, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0007, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.20552389323711395, "learning_rate": 6.940176563004123e-05, "loss": 0.0012999398168176413, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0013, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.8873184323310852, "learning_rate": 6.912644503343682e-05, "loss": 0.014279229566454887, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.01438, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.3493788242340088, "learning_rate": 6.885053657779273e-05, "loss": 0.006735142320394516, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00676, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 20.96, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.20922653377056122, "learning_rate": 6.857405174478604e-05, "loss": 0.003210448892787099, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00322, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.9769893884658813, "learning_rate": 6.82970020400792e-05, "loss": 0.010232124477624893, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01028, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.49, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 1.0841566324234009, "learning_rate": 6.801939899284132e-05, "loss": 0.003384920069947839, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.00339, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.23099999129772186, "learning_rate": 6.774125415526827e-05, "loss": 0.003550637746229768, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00356, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.24, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 1.1071275472640991, "learning_rate": 6.746257910210214e-05, "loss": 0.007122470065951347, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00715, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.18913541734218597, "learning_rate": 6.718338543014937e-05, "loss": 0.0030192467384040356, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00302, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 33.73, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.35436588525772095, "learning_rate": 6.69036847577983e-05, "loss": 0.005669867619872093, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.00569, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.05735715851187706, "learning_rate": 6.662348872453553e-05, "loss": 0.0006210830761119723, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00062, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.2853192090988159, "learning_rate": 6.63428089904618e-05, "loss": 0.0025981655344367027, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.0026, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.5234872698783875, "learning_rate": 6.60616572358065e-05, "loss": 0.010506541468203068, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01056, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.56, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.6833733916282654, "learning_rate": 6.578004516044172e-05, "loss": 0.004140924662351608, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.00415, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.250273197889328, "learning_rate": 6.549798448339548e-05, "loss": 0.0016205488936975598, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00162, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.8714818358421326, "learning_rate": 6.521548694236384e-05, "loss": 0.019340110942721367, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.01953, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.4514566957950592, "learning_rate": 6.493256429322259e-05, "loss": 0.008509005419909954, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00855, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.26371243596076965, "learning_rate": 6.464922830953799e-05, "loss": 0.008254537358880043, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00829, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.6582297086715698, "learning_rate": 6.436549078207688e-05, "loss": 0.0024804507847875357, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.00248, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.09737692028284073, "learning_rate": 6.408136351831592e-05, "loss": 0.0010533290915191174, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00105, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.44, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.5441069006919861, "learning_rate": 6.379685834195036e-05, "loss": 0.002414551330730319, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00242, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.5757384300231934, "learning_rate": 6.351198709240186e-05, "loss": 0.005687021184712648, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.0057, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 1.3602705001831055, "learning_rate": 6.32267616243259e-05, "loss": 0.01998007297515869, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.02018, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.10490329563617706, "learning_rate": 6.294119380711849e-05, "loss": 0.0008182218298316002, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00082, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.0962253138422966, "learning_rate": 6.265529552442209e-05, "loss": 0.0009877507109194994, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00099, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.32850420475006104, "learning_rate": 6.236907867363127e-05, "loss": 0.010325739160180092, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01038, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.11584971100091934, "learning_rate": 6.208255516539749e-05, "loss": 0.001304705860093236, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00131, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.5319051146507263, "learning_rate": 6.179573692313344e-05, "loss": 0.007187267299741507, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00721, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.11785684525966644, "learning_rate": 6.150863588251694e-05, "loss": 0.0013930774293839931, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00139, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.09003490209579468, "learning_rate": 6.122126399099419e-05, "loss": 0.0005670200334861875, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00057, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.3325178027153015, "learning_rate": 6.0933633207282615e-05, "loss": 0.0020435932092368603, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00205, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 2.0883212089538574, "learning_rate": 6.064575550087316e-05, "loss": 0.008298702538013458, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.00833, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.35318636894226074, "learning_rate": 6.0357642851532245e-05, "loss": 0.0038977107033133507, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00391, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.012203142046928406, "learning_rate": 6.0069307248803294e-05, "loss": 0.0001653638610150665, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00017, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.46, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.15504305064678192, "learning_rate": 5.9780760691507635e-05, "loss": 0.001979721011593938, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00198, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.14558397233486176, "learning_rate": 5.9492015187245334e-05, "loss": 0.0012645245296880603, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00127, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.3749781548976898, "learning_rate": 5.920308275189541e-05, "loss": 0.0046793008223176, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.00469, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.14241816103458405, "learning_rate": 5.8913975409115874e-05, "loss": 0.0016404457855969667, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00164, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.23385171592235565, "learning_rate": 5.8624705189843395e-05, "loss": 0.0010672437492758036, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00107, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.1722296178340912, "learning_rate": 5.833528413179249e-05, "loss": 0.0013754154788330197, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00138, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.4, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 0.529482901096344, "learning_rate": 5.80457242789548e-05, "loss": 0.005757027305662632, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00577, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.4409250319004059, "learning_rate": 5.77560376810977e-05, "loss": 0.003099203109741211, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.0031, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.22375717759132385, "learning_rate": 5.7466236393263005e-05, "loss": 0.008542000316083431, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00858, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.24301159381866455, "learning_rate": 5.717633247526522e-05, "loss": 0.0014905647840350866, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00149, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.10225414484739304, "learning_rate": 5.688633799118971e-05, "loss": 0.0012149509275332093, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00122, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 1.238974690437317, "learning_rate": 5.659626500889066e-05, "loss": 0.002359751146286726, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00236, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 30.67, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 0.4416565001010895, "learning_rate": 5.6306125599488905e-05, "loss": 0.004353099036961794, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.00436, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.3350789546966553, "learning_rate": 5.601593183686955e-05, "loss": 0.002483048941940069, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.00249, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.35, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 0.005217446479946375, "learning_rate": 5.572569579717961e-05, "loss": 8.285699732368812e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00008, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 0.017666663974523544, "learning_rate": 5.543542955832538e-05, "loss": 9.225462417816743e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00009, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.3687189519405365, "learning_rate": 5.514514519946986e-05, "loss": 0.003413026686757803, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00342, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.031272195279598236, "learning_rate": 5.485485480053015e-05, "loss": 0.00016663990390952677, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00017, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.3517431318759918, "learning_rate": 5.4564570441674645e-05, "loss": 0.0005368043202906847, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00054, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.8265377283096313, "learning_rate": 5.42743042028204e-05, "loss": 0.0037325031589716673, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00374, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.5904502272605896, "learning_rate": 5.3984068163130464e-05, "loss": 0.005233262665569782, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00525, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 22.75, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.2786749005317688, "learning_rate": 5.369387440051111e-05, "loss": 0.001960987923666835, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00196, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.23101817071437836, "learning_rate": 5.340373499110935e-05, "loss": 0.0025752901565283537, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00258, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.10081073641777039, "learning_rate": 5.3113662008810304e-05, "loss": 0.0005547528271563351, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00055, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.010565731674432755, "learning_rate": 5.282366752473479e-05, "loss": 9.580043115420267e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.0001, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 1.1300185918807983, "learning_rate": 5.2533763606737005e-05, "loss": 0.010617226362228394, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.01067, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 0.02684837393462658, "learning_rate": 5.224396231890232e-05, "loss": 0.0001537603820906952, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00015, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 1.1142722368240356, "learning_rate": 5.195427572104522e-05, "loss": 0.0012748180888593197, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00128, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.003483360633254051, "learning_rate": 5.166471586820751e-05, "loss": 1.7817812477005646e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00002, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.008755158632993698, "learning_rate": 5.1375294810156615e-05, "loss": 5.363760283216834e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00005, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 0.20352716743946075, "learning_rate": 5.1086024590884144e-05, "loss": 0.00043158623157069087, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00043, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.004675888456404209, "learning_rate": 5.079691724810461e-05, "loss": 3.639070564531721e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00004, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.7542117834091187, "learning_rate": 5.0507984812754684e-05, "loss": 0.01341431587934494, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0135, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.004134606570005417, "learning_rate": 5.021923930849237e-05, "loss": 3.64735460607335e-05, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00004, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.018427778035402298, "learning_rate": 4.99306927511967e-05, "loss": 0.0002534704399295151, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00025, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.7422452569007874, "learning_rate": 4.964235714846775e-05, "loss": 0.008427763357758522, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00846, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.04031055420637131, "learning_rate": 4.9354244499126866e-05, "loss": 0.00023869781580287963, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00024, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.32818761467933655, "learning_rate": 4.90663667927174e-05, "loss": 0.002676003146916628, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00268, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 1.2155787944793701, "learning_rate": 4.877873600900581e-05, "loss": 0.03164320066571236, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.03215, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.9833851456642151, "learning_rate": 4.849136411748306e-05, "loss": 0.004906849469989538, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00492, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.02746356464922428, "learning_rate": 4.8204263076866574e-05, "loss": 0.00015035917749628425, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00015, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.30956315994262695, "learning_rate": 4.791744483460251e-05, "loss": 0.0011974794324487448, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.0012, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.32648026943206787, "learning_rate": 4.7630921326368736e-05, "loss": 0.003914389759302139, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00392, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 19.94, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.37184473872184753, "learning_rate": 4.7344704475577916e-05, "loss": 0.004815159365534782, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.00483, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.391240656375885, "learning_rate": 4.705880619288153e-05, "loss": 0.00482224952429533, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00483, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.10286086052656174, "learning_rate": 4.677323837567412e-05, "loss": 0.0010607833974063396, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00106, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.4797014892101288, "learning_rate": 4.6488012907598146e-05, "loss": 0.005354643799364567, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00537, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.24, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.4420396089553833, "learning_rate": 4.620314165804964e-05, "loss": 0.00560842826962471, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00562, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.5881717801094055, "learning_rate": 4.591863648168407e-05, "loss": 0.006422752980142832, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00644, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.43312492966651917, "learning_rate": 4.5634509217923135e-05, "loss": 0.0024784316774457693, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00248, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 31.01, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 1.3325884342193604, "learning_rate": 4.535077169046201e-05, "loss": 0.008415870368480682, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.00845, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.3158382177352905, "learning_rate": 4.506743570677743e-05, "loss": 0.004468950908631086, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00448, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.14678345620632172, "learning_rate": 4.478451305763618e-05, "loss": 0.002875756472349167, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00288, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.06776336580514908, "learning_rate": 4.450201551660454e-05, "loss": 0.0012549967505037785, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00126, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.0958128571510315, "learning_rate": 4.4219954839558276e-05, "loss": 0.0016585225239396095, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.00166, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.24466221034526825, "learning_rate": 4.393834276419352e-05, "loss": 0.0009206987451761961, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00092, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.053055983036756516, "learning_rate": 4.36571910095382e-05, "loss": 0.000602126179728657, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0006, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.10594559460878372, "learning_rate": 4.337651127546448e-05, "loss": 0.0008799374336376786, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00088, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.1705891191959381, "learning_rate": 4.3096315242201736e-05, "loss": 0.0012520075542852283, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00125, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 31.05, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.01903805136680603, "learning_rate": 4.2816614569850635e-05, "loss": 0.0003125502844341099, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.00031, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 1.0169895887374878, "learning_rate": 4.2537420897897864e-05, "loss": 0.015176085755228996, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01529, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.24, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.030921433120965958, "learning_rate": 4.225874584473174e-05, "loss": 0.000404372694902122, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0004, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.07108241319656372, "learning_rate": 4.19806010071587e-05, "loss": 0.0009299074299633503, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00093, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.05730182304978371, "learning_rate": 4.170299795992081e-05, "loss": 0.0005536397802643478, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00055, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.33336886763572693, "learning_rate": 4.142594825521398e-05, "loss": 0.0017714172136038542, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00177, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 1.1850405931472778, "learning_rate": 4.114946342220728e-05, "loss": 0.0003953118284698576, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.0004, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.012935487553477287, "learning_rate": 4.087355496656321e-05, "loss": 0.00013684491568710655, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00014, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.08023589104413986, "learning_rate": 4.05982343699588e-05, "loss": 0.0005785867106169462, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00058, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.17765015363693237, "learning_rate": 4.0323513089607876e-05, "loss": 0.0010662261629477143, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00107, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.017262451350688934, "learning_rate": 4.004940255778431e-05, "loss": 0.00013672554632648826, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00014, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.44427093863487244, "learning_rate": 3.977591418134619e-05, "loss": 0.002564815804362297, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00257, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.18982231616973877, "learning_rate": 3.95030593412612e-05, "loss": 0.0012563832569867373, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00126, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.39, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.8500872254371643, "learning_rate": 3.923084939213296e-05, "loss": 0.0052078235894441605, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00522, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 23.81, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.24542823433876038, "learning_rate": 3.895929566172861e-05, "loss": 0.001535475137643516, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00154, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.36, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.004411033354699612, "learning_rate": 3.868840945050728e-05, "loss": 8.230144158005714e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00008, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 1.2411470413208008, "learning_rate": 3.841820203114995e-05, "loss": 0.012582246214151382, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.01266, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.06485675275325775, "learning_rate": 3.814868464809027e-05, "loss": 0.000140393094625324, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00014, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.40247395634651184, "learning_rate": 3.787986851704667e-05, "loss": 0.005231763701885939, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00525, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.14647816121578217, "learning_rate": 3.7611764824555654e-05, "loss": 0.0008264593197964132, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00083, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.28573718667030334, "learning_rate": 3.734438472750619e-05, "loss": 0.0018234081799164414, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00183, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.09553559869527817, "learning_rate": 3.707773935267552e-05, "loss": 0.0005577140836976469, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00056, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.00782093033194542, "learning_rate": 3.68118397962661e-05, "loss": 0.00011736310989363119, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00012, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.4244482219219208, "learning_rate": 3.654669712344384e-05, "loss": 0.017745811492204666, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.0179, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.36, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.024637557566165924, "learning_rate": 3.628232236787763e-05, "loss": 8.936172525864094e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00009, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.9742305278778076, "learning_rate": 3.6018726531280144e-05, "loss": 0.002953664632514119, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.00296, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 0.8469567894935608, "learning_rate": 3.575592058295017e-05, "loss": 0.00994935818016529, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.27767297625541687, "learning_rate": 3.549391545931585e-05, "loss": 0.003351568942889571, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00336, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.3081471621990204, "learning_rate": 3.5232722063479914e-05, "loss": 0.0020967326126992702, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.0021, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.023037493228912354, "learning_rate": 3.49723512647657e-05, "loss": 0.000256448081927374, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00026, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.22291229665279388, "learning_rate": 3.471281389826491e-05, "loss": 0.0004476486355997622, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00045, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.06171007826924324, "learning_rate": 3.4454120764386764e-05, "loss": 0.00045065765152685344, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00045, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.03629618138074875, "learning_rate": 3.4196282628408526e-05, "loss": 0.000393298570998013, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00039, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.17715299129486084, "learning_rate": 3.3939310220027456e-05, "loss": 0.00033384276321157813, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00033, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.67, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.10879218578338623, "learning_rate": 3.3683214232914404e-05, "loss": 0.0013005926739424467, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.0013, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.1630091518163681, "learning_rate": 3.342800532426873e-05, "loss": 0.0008514306391589344, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.00085, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.01525947730988264, "learning_rate": 3.317369411437484e-05, "loss": 8.847589924698696e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00009, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.06235525384545326, "learning_rate": 3.292029118616024e-05, "loss": 0.0008286050288006663, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00083, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.13583293557167053, "learning_rate": 3.266780708475511e-05, "loss": 0.001037516864016652, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00104, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.18, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.40652626752853394, "learning_rate": 3.241625231705354e-05, "loss": 0.0003939187154173851, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00039, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 0.5584993362426758, "learning_rate": 3.216563735127618e-05, "loss": 0.0034244470298290253, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00343, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.20709070563316345, "learning_rate": 3.191597261653475e-05, "loss": 0.002132589928805828, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00213, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 0.1983594000339508, "learning_rate": 3.166726850239794e-05, "loss": 0.002521308371797204, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00252, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.25, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.025563286617398262, "learning_rate": 3.141953535845912e-05, "loss": 0.00025044698850251734, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00025, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.26974013447761536, "learning_rate": 3.11727834939056e-05, "loss": 0.005847956985235214, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00587, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.09549195319414139, "learning_rate": 3.092702317708967e-05, "loss": 0.0012451761867851019, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00125, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 0.10553739964962006, "learning_rate": 3.0682264635101276e-05, "loss": 0.00013774196850135922, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.00014, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.1663064807653427, "learning_rate": 3.0438518053342407e-05, "loss": 0.0008104329463094473, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00081, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.5360269546508789, "learning_rate": 3.0195793575103266e-05, "loss": 0.00602231128141284, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00604, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.07, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.49787482619285583, "learning_rate": 2.9954101301140146e-05, "loss": 0.008917278610169888, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.00896, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 21.93, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.7666929364204407, "learning_rate": 2.9713451289255123e-05, "loss": 0.012657465413212776, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.01274, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.5313919186592102, "learning_rate": 2.9473853553877484e-05, "loss": 0.007152163423597813, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00718, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.17941053211688995, "learning_rate": 2.9235318065647e-05, "loss": 0.000780510192271322, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00078, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.7784935235977173, "learning_rate": 2.8997854750998964e-05, "loss": 0.008989064954221249, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00903, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.060946281999349594, "learning_rate": 2.8761473491751258e-05, "loss": 0.00037636555498465896, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00038, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.06742117553949356, "learning_rate": 2.8526184124692883e-05, "loss": 0.000768409576267004, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00077, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.029117247089743614, "learning_rate": 2.829199644117484e-05, "loss": 0.0005752278957515955, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00058, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.016692642122507095, "learning_rate": 2.8058920186702553e-05, "loss": 0.0002644261112436652, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00026, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.04412776604294777, "learning_rate": 2.782696506053033e-05, "loss": 0.0004822240734938532, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00048, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.06203721463680267, "learning_rate": 2.7596140715257824e-05, "loss": 0.0006452169036492705, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00065, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 21.89, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.04308488965034485, "learning_rate": 2.7366456756428184e-05, "loss": 0.000514882558491081, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00052, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.1432814598083496, "learning_rate": 2.7137922742128486e-05, "loss": 0.0014066064031794667, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00141, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 29.98, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.15579921007156372, "learning_rate": 2.691054818259188e-05, "loss": 0.002079858910292387, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00208, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.036354463547468185, "learning_rate": 2.6684342539801933e-05, "loss": 0.0002997233241330832, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.0003, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.007968918420374393, "learning_rate": 2.645931522709877e-05, "loss": 0.00013599595695268363, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00014, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.42, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.016158074140548706, "learning_rate": 2.6235475608787365e-05, "loss": 0.00016321682778652757, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00016, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.11980169266462326, "learning_rate": 2.6012832999747916e-05, "loss": 0.0012514974223449826, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00125, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.0901523008942604, "learning_rate": 2.579139666504821e-05, "loss": 0.001516394317150116, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00152, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.0856979563832283, "learning_rate": 2.557117581955798e-05, "loss": 0.0004326904017943889, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00043, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.02404644526541233, "learning_rate": 2.5352179627565532e-05, "loss": 0.00021640419436153024, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00022, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.524491012096405, "learning_rate": 2.5134417202396277e-05, "loss": 0.007519315928220749, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00755, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.11914055049419403, "learning_rate": 2.491789760603361e-05, "loss": 0.0015392648056149483, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00154, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.023861847817897797, "learning_rate": 2.4702629848741764e-05, "loss": 0.00023304611386265606, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00023, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.18, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.027571724727749825, "learning_rate": 2.4488622888690785e-05, "loss": 0.00033765024272724986, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00034, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.08315709233283997, "learning_rate": 2.427588563158384e-05, "loss": 0.0004644090076908469, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00046, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.026718847453594208, "learning_rate": 2.406442693028651e-05, "loss": 0.0004534275794867426, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00045, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.10161672532558441, "learning_rate": 2.3854255584458547e-05, "loss": 0.0007325086044147611, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00073, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.058808743953704834, "learning_rate": 2.3645380340187508e-05, "loss": 0.0003809864865615964, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00038, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.10522804409265518, "learning_rate": 2.3437809889624914e-05, "loss": 0.0011356035247445107, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00114, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.046325623989105225, "learning_rate": 2.3231552870624487e-05, "loss": 0.0002346369146835059, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00023, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.4839786887168884, "learning_rate": 2.3026617866382657e-05, "loss": 0.0015968342777341604, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.0016, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.43359440565109253, "learning_rate": 2.2823013405081507e-05, "loss": 0.011573181487619877, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01164, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.51, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.009161097928881645, "learning_rate": 2.2620747959533722e-05, "loss": 8.7327032815665e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00009, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.66, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.512662947177887, "learning_rate": 2.2419829946830123e-05, "loss": 0.004903641529381275, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00492, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 2.9679219722747803, "learning_rate": 2.2220267727989325e-05, "loss": 0.0028094665613025427, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.00281, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.18253013491630554, "learning_rate": 2.202206960760984e-05, "loss": 0.0009201104985550046, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.00092, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.01953180320560932, "learning_rate": 2.182524383352446e-05, "loss": 0.00015365466242656112, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00015, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.010517308488488197, "learning_rate": 2.1629798596457056e-05, "loss": 0.00014398334315046668, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00014, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 31.66, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.1940128058195114, "learning_rate": 2.1435742029681725e-05, "loss": 0.0024753331672400236, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.00248, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.0496944859623909, "learning_rate": 2.124308220868431e-05, "loss": 0.0007137508364394307, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00071, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.3683420717716217, "learning_rate": 2.105182715082638e-05, "loss": 0.0009208200499415398, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00092, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.023274550214409828, "learning_rate": 2.0861984815011552e-05, "loss": 0.0001416395971318707, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00014, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.14180342853069305, "learning_rate": 2.0673563101354323e-05, "loss": 0.00024280388606712222, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00024, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.016201617196202278, "learning_rate": 2.0486569850851317e-05, "loss": 0.00021794547501485795, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00022, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.03290703147649765, "learning_rate": 2.0301012845054956e-05, "loss": 0.0003092651895713061, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00031, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 0.05167189612984657, "learning_rate": 2.011689980574966e-05, "loss": 0.00026560892001725733, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00027, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.5224520564079285, "learning_rate": 1.993423839463052e-05, "loss": 0.008841147646307945, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00888, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 0.14875978231430054, "learning_rate": 1.975303621298445e-05, "loss": 0.0008162078447639942, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.00082, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.015263130888342857, "learning_rate": 1.957330080137385e-05, "loss": 0.00015176778833847493, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00015, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.015147769823670387, "learning_rate": 1.9395039639322864e-05, "loss": 0.0001447954709874466, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00014, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.01550917886197567, "learning_rate": 1.9218260145006073e-05, "loss": 0.00013999728253111243, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00014, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.28, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.06268800795078278, "learning_rate": 1.904296967493982e-05, "loss": 0.0006348071037791669, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00064, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 29.02, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 0.2498202919960022, "learning_rate": 1.8869175523676064e-05, "loss": 0.0022089178673923016, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00221, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.04775214195251465, "learning_rate": 1.869688492349885e-05, "loss": 0.0005830471054650843, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00058, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.2513720691204071, "learning_rate": 1.85261050441233e-05, "loss": 0.002165381098166108, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00217, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 22.86, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.015338403172791004, "learning_rate": 1.8356842992397304e-05, "loss": 0.00011308112880215049, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00011, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.02904435805976391, "learning_rate": 1.8189105812005714e-05, "loss": 2.921275154221803e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00003, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.017892900854349136, "learning_rate": 1.802290048317732e-05, "loss": 0.0001938966743182391, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00019, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.0048964316956698895, "learning_rate": 1.785823392239424e-05, "loss": 6.3496278016828e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00006, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 1.9382976293563843, "learning_rate": 1.7695112982104225e-05, "loss": 0.007037092465907335, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00706, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.0016038189642131329, "learning_rate": 1.7533544450435433e-05, "loss": 1.4564624507329427e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00001, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.05714528635144234, "learning_rate": 1.7373535050913946e-05, "loss": 0.00038622418651357293, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00039, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.003173970617353916, "learning_rate": 1.721509144218405e-05, "loss": 5.076732486486435e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00005, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.09898708760738373, "learning_rate": 1.705822021773101e-05, "loss": 0.00027678709011524916, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00028, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.018864842131733894, "learning_rate": 1.69029279056068e-05, "loss": 0.00012027710909023881, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00012, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.019368667155504227, "learning_rate": 1.6749220968158415e-05, "loss": 0.00014314577856566757, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00014, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.005447172559797764, "learning_rate": 1.659710580175893e-05, "loss": 5.873618647456169e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.00006, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.11761007457971573, "learning_rate": 1.644658873654133e-05, "loss": 0.0006599801708944142, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00066, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.050796039402484894, "learning_rate": 1.629767603613508e-05, "loss": 0.00011276356963207945, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00011, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.2624942660331726, "learning_rate": 1.615037389740547e-05, "loss": 0.001476677367463708, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00148, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 0.007726503070443869, "learning_rate": 1.600468845019576e-05, "loss": 2.2926502424525097e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.00002, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.45, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.5729708671569824, "learning_rate": 1.5860625757072092e-05, "loss": 0.005049503408372402, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00506, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.0419280044734478, "learning_rate": 1.571819181307116e-05, "loss": 0.0003502543841022998, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00035, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.052264198660850525, "learning_rate": 1.557739254545075e-05, "loss": 0.00025272706989198923, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00025, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.010161260142922401, "learning_rate": 1.543823381344311e-05, "loss": 8.921208063839003e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00009, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.03831199184060097, "learning_rate": 1.5300721408011114e-05, "loss": 8.548055484425277e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00009, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.01041450072079897, "learning_rate": 1.5164861051607254e-05, "loss": 6.861635483801365e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00007, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.4, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.25208088755607605, "learning_rate": 1.5030658397935521e-05, "loss": 0.0010729912901297212, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00107, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.0029376933816820383, "learning_rate": 1.4898119031716104e-05, "loss": 2.1667325199814513e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00002, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.1, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.006010542158037424, "learning_rate": 1.476724846845306e-05, "loss": 6.0429261793615296e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00006, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.39, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 0.360151082277298, "learning_rate": 1.463805215420471e-05, "loss": 0.0011959499679505825, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.0012, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.13575150072574615, "learning_rate": 1.451053546535705e-05, "loss": 0.00010766807827167213, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00011, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.015444664284586906, "learning_rate": 1.438470370840001e-05, "loss": 5.064180732006207e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00005, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.07490239292383194, "learning_rate": 1.4260562119706606e-05, "loss": 0.0008251686813309789, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00083, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.00661479402333498, "learning_rate": 1.413811586531508e-05, "loss": 3.265037594246678e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00003, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.68, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.03475445508956909, "learning_rate": 1.4017370040713884e-05, "loss": 0.0002443317789584398, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00024, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.2824845016002655, "learning_rate": 1.3898329670629645e-05, "loss": 0.0012723141117021441, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00127, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.37, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 0.020561836659908295, "learning_rate": 1.3780999708818058e-05, "loss": 0.00014706453657709062, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00015, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.00256377668119967, "learning_rate": 1.3665385037857758e-05, "loss": 2.8685950383078307e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00003, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.10538701713085175, "learning_rate": 1.3551490468947126e-05, "loss": 0.0005012988112866879, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.0005, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.6513829231262207, "learning_rate": 1.3439320741704075e-05, "loss": 0.012066565454006195, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.01214, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.010520837269723415, "learning_rate": 1.3328880523968808e-05, "loss": 5.7916593505069613e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00006, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.061182331293821335, "learning_rate": 1.3220174411609587e-05, "loss": 0.00042716742609627545, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00043, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.07, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.057951100170612335, "learning_rate": 1.3113206928331471e-05, "loss": 0.00028185819974169135, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00028, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.35518592596054077, "learning_rate": 1.300798252548806e-05, "loss": 0.0011299459729343653, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00113, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.010331215336918831, "learning_rate": 1.2904505581896265e-05, "loss": 9.20655220397748e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00009, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.8672090768814087, "learning_rate": 1.2802780403654082e-05, "loss": 0.016763780266046524, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.01691, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.2574054002761841, "learning_rate": 1.2702811223961408e-05, "loss": 0.00065491913119331, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.00066, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.057762742042541504, "learning_rate": 1.2604602202943861e-05, "loss": 0.0005738360341638327, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00057, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.09155192226171494, "learning_rate": 1.2508157427479686e-05, "loss": 0.0004957674536854029, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.0005, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.011142891831696033, "learning_rate": 1.2413480911029655e-05, "loss": 0.00012845003220718354, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00013, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.40677809715270996, "learning_rate": 1.2320576593470082e-05, "loss": 0.004871611017733812, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00488, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.30363738536834717, "learning_rate": 1.2229448340928828e-05, "loss": 0.003776694182306528, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00378, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.09424100816249847, "learning_rate": 1.2140099945624458e-05, "loss": 0.0008757114992477, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00088, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.002041307743638754, "learning_rate": 1.205253512570841e-05, "loss": 2.946273707493674e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00003, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.7013693451881409, "learning_rate": 1.1966757525110255e-05, "loss": 0.0035451059229671955, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00355, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.022810503840446472, "learning_rate": 1.1882770713386095e-05, "loss": 0.00026283672195859253, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00026, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.40189114212989807, "learning_rate": 1.180057818556998e-05, "loss": 0.0023606442846357822, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00236, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.7969369888305664, "learning_rate": 1.1720183362028494e-05, "loss": 0.012694726698100567, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.01278, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.0024883418809622526, "learning_rate": 1.1641589588318387e-05, "loss": 4.002712739747949e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00004, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.196363165974617, "learning_rate": 1.1564800135047418e-05, "loss": 0.0006277322536334395, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00063, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.014423682354390621, "learning_rate": 1.148981819773816e-05, "loss": 0.00013483656221069396, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00013, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.1896059662103653, "learning_rate": 1.1416646896695086e-05, "loss": 0.0017132106004282832, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00171, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.3545624613761902, "learning_rate": 1.1345289276874717e-05, "loss": 0.004747453145682812, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00476, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.7811309099197388, "learning_rate": 1.1275748307758873e-05, "loss": 0.035745128989219666, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.03639, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 22.72, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.008582853712141514, "learning_rate": 1.1208026883231147e-05, "loss": 7.10546737536788e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00007, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.032635606825351715, "learning_rate": 1.1142127821456433e-05, "loss": 0.00025895837461575866, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00026, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.030166443437337875, "learning_rate": 1.1078053864763674e-05, "loss": 0.0003453365934547037, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00035, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 0.02498539723455906, "learning_rate": 1.1015807679531756e-05, "loss": 9.928976214723662e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.0001, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.30655354261398315, "learning_rate": 1.0955391856078528e-05, "loss": 0.0013873155694454908, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00139, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.8158500790596008, "learning_rate": 1.0896808908553007e-05, "loss": 0.013315004296600819, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0134, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.3007687032222748, "learning_rate": 1.0840061274830763e-05, "loss": 0.0009049343643710017, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.00091, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.30412137508392334, "learning_rate": 1.0785151316412473e-05, "loss": 0.0015199125045910478, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00152, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.02716813236474991, "learning_rate": 1.0732081318325639e-05, "loss": 0.0002143428719136864, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00021, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.5038143396377563, "learning_rate": 1.0680853489029501e-05, "loss": 0.0025534722954034805, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00256, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.006719888653606176, "learning_rate": 1.0631469960323152e-05, "loss": 9.227527334587649e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00009, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.19248120486736298, "learning_rate": 1.0583932787256783e-05, "loss": 0.0019882898777723312, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00199, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.07443113625049591, "learning_rate": 1.0538243948046206e-05, "loss": 0.0004588438314385712, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.00046, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.007754078600555658, "learning_rate": 1.0494405343990523e-05, "loss": 0.00012373751087579876, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00012, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.13313932716846466, "learning_rate": 1.0452418799392985e-05, "loss": 0.0008159930002875626, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00082, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 1.6239131689071655, "learning_rate": 1.0412286061485102e-05, "loss": 0.008568096905946732, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.0086, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.004006825853139162, "learning_rate": 1.03740088003539e-05, "loss": 3.482437750790268e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00003, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.5532016754150391, "learning_rate": 1.0337588608872463e-05, "loss": 0.002880650572478771, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00288, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 0.5473046898841858, "learning_rate": 1.0303027002633622e-05, "loss": 0.008618625812232494, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00866, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.013172691687941551, "learning_rate": 1.0270325419886884e-05, "loss": 0.0002160731964977458, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00022, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.02131938375532627, "learning_rate": 1.0239485221478599e-05, "loss": 9.844663145486265e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.0001, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.11334938555955887, "learning_rate": 1.0210507690795292e-05, "loss": 0.0007168808951973915, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00072, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.024323835968971252, "learning_rate": 1.0183394033710305e-05, "loss": 0.00028721155831590295, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00029, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.011693691834807396, "learning_rate": 1.0158145378533583e-05, "loss": 0.0001215750235132873, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00012, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.060612987726926804, "learning_rate": 1.0134762775964726e-05, "loss": 0.0007441648631356657, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00074, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.2102545201778412, "learning_rate": 1.0113247199049278e-05, "loss": 0.001090665115043521, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00109, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.029205728322267532, "learning_rate": 1.0093599543138205e-05, "loss": 0.00014751346316188574, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00015, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.007648298516869545, "learning_rate": 1.0075820625850675e-05, "loss": 0.0001227802858920768, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00012, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.12508590519428253, "learning_rate": 1.0059911187040013e-05, "loss": 0.0008103963918983936, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00081, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.453528493642807, "learning_rate": 1.0045871888762893e-05, "loss": 0.0012893046950921416, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00129, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.026012606918811798, "learning_rate": 1.003370331525184e-05, "loss": 0.00020336048328317702, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.0002, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.04012585058808327, "learning_rate": 1.002340597289085e-05, "loss": 0.0002895949291996658, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00029, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.09142807126045227, "learning_rate": 1.0014980290194387e-05, "loss": 0.0009094822453334928, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00091, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.0633169636130333, "learning_rate": 1.0008426617789489e-05, "loss": 0.00081396393943578, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00081, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.08104396611452103, "learning_rate": 1.0003745228401215e-05, "loss": 0.000576217717025429, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00058, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.06120160594582558, "learning_rate": 1.0000936316841296e-05, "loss": 0.0007413307903334498, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00074, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 234252} diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..958cc312830fc1897312806c81815e46f898b543 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A1-12b-1/parents/gemma3_12b_1m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..01efba4f9fdf3ae88ef97da34ed954976058545b --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 05:37:43,551] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 05:37:46.623000 48759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:37:46.644000 48759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 05:37:49,053] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:37:49,055] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 05:37:49,056] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:37:49,079] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 05:37:49,080] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:37:49,081] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 05:37:49,134] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 05:37:49,135] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 05:37:49,171] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-12b-1/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 05:37:51,642] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 05:37:51,642] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 05:37:51,642] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 22514.27 examples/s] Generating train split: 7980 examples [00:00, 25294.01 examples/s] Generating train split: 8192 examples [00:00, 24413.80 examples/s] +[2026-09-08 05:37:53,036] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 05:37:53,038] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6319.41 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25216.34 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.048000 49036 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.065000 49029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.076000 49031 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.089000 49029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.097000 49031 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.142000 49032 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.148000 49030 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.164000 49032 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 05:38:35.170000 49030 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 246.63 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 246.63 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 246.63 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 246.63 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 246.63 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 862.99 examples/s] +[2026-09-08 05:38:38,946] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 05:38:47,856] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 05:38:55,175] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 05:39:03,735] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/checkpoints... +[2026-09-08 05:39:03,736] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/checkpoints... +[2026-09-08 05:39:04,067] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/checkpoints... +[2026-09-08 05:39:04,072] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_charter/train/checkpoints... +[2026-09-08 05:39:04,423] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f0da50078316544524c6ae5f8a9a8dc570392c3b --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,847 @@ +[2026-09-08 03:48:07,079] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 03:48:10.472000 42457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:48:10.500000 42457 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 03:48:13,201] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 03:48:13,204] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 03:48:13,204] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 03:48:13,235] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 03:48:13,235] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 03:48:13,236] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 03:48:13,292] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 03:48:13,292] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 03:48:13,372] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-12b-1/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 03:48:16,070] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 03:48:16,070] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 03:48:16,070] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 33229.15 examples/s] Generating train split: 7980 examples [00:00, 29383.70 examples/s] Generating train split: 8192 examples [00:00, 29318.81 examples/s] +[2026-09-08 03:48:17,379] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 03:48:17,382] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6556.35 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 25475.79 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:00.997000 42710 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:01.042000 42717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:01.044000 42713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:01.061000 42711 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:01.063000 42717 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:01.066000 42713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:01.083000 42711 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:01.088000 42712 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 03:49:01.111000 42712 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 248.46 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 248.46 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 248.46 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 248.46 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 248.46 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 873.78 examples/s] +[2026-09-08 03:49:05,016] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 03:49:15,029] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 03:49:20,847] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 03:49:29,436] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/checkpoints... +[2026-09-08 03:49:29,436] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/checkpoints... +[2026-09-08 03:49:29,788] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/checkpoints... +[2026-09-08 03:49:29,795] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-12b-1/cells/gemma3_12b_1m/charter/mixed_coin/train/checkpoints... +[2026-09-08 03:49:30,124] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d2349d343c8adccd514c5f307de9df9a187971a5 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 20:27:14,052] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 20:27:17.195000 16822 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:27:17.217000 16822 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 20:27:19,671] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:27:19,673] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 20:27:19,674] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:27:19,697] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 20:27:19,699] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:27:19,700] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 20:27:19,754] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 20:27:19,754] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 20:27:19,791] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-12b-1/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 20:27:22,244] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/prepared/7af732f447ea1291379b9572cc239e7a +[2026-09-07 20:27:22,244] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 20:27:22,244] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 33011.98 examples/s] Generating train split: 7974 examples [00:00, 30673.72 examples/s] Generating train split: 8192 examples [00:00, 30278.69 examples/s] +[2026-09-07 20:27:23,508] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 20:27:23,510] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5731.24 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 24921.55 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.664000 17071 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.721000 17065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.743000 17065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.762000 17064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.776000 17067 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.783000 17064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.796000 17067 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.796000 17066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 20:28:05.817000 17066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 249.46 examples/s] Saving the dataset (1/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 249.46 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 249.46 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 249.46 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 249.46 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 876.48 examples/s] +[2026-09-07 20:28:09,577] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 20:28:18,592] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 20:28:26,003] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 20:28:35,556] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/checkpoints... +[2026-09-07 20:28:35,556] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/checkpoints... +[2026-09-07 20:28:35,925] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/checkpoints... +[2026-09-07 20:28:35,933] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_1pct/train/checkpoints... +[2026-09-07 20:28:36,268] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..718be4c7e8bab76b0605856e157cd2b73ab9568b --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 22:16:57,039] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 22:17:00.081000 23487 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:00.103000 23487 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 22:17:02,463] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 22:17:02,465] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 22:17:02,465] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 22:17:02,489] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 22:17:02,490] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 22:17:02,491] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 22:17:02,546] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 22:17:02,546] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 22:17:02,585] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-12b-1/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 22:17:05,075] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/prepared/25f3217c00f2c003854a38d88cde4036 +[2026-09-07 22:17:05,075] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 22:17:05,075] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 30867.58 examples/s] Generating train split: 7992 examples [00:00, 31252.71 examples/s] Generating train split: 8192 examples [00:00, 30608.57 examples/s] +[2026-09-07 22:17:06,339] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 22:17:06,342] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6947.47 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 27362.80 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:47.734000 23728 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:47.754000 23729 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:47.756000 23728 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:47.849000 23727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:47.851000 23734 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:47.872000 23727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:47.873000 23734 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:47.981000 23730 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 22:17:48.003000 23730 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 249.29 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 249.29 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 249.29 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 249.29 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 249.29 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 876.22 examples/s] +[2026-09-07 22:17:51,586] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 22:18:00,515] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 22:18:07,867] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 22:18:16,331] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/checkpoints... +[2026-09-07 22:18:16,332] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/checkpoints... +[2026-09-07 22:18:16,679] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/checkpoints... +[2026-09-07 22:18:16,686] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/charter_5pct/train/checkpoints... +[2026-09-07 22:18:17,024] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..29219bacb9ee1dcad640c336852f5eeacbc1029d --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 16:47:13,656] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 16:47:20,195] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 16:47:20.412000 2133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:47:20.491000 2133 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 16:47:25,878] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 16:47:25,883] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 16:47:25,883] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 16:47:25,912] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 16:47:25,912] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 16:47:25,914] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 16:47:25,968] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 16:47:25,968] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 16:47:26,284] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-12b-1/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 16:47:31,629] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/prepared/33c3dd430eb18b87b0867c58253f2fd3 +[2026-09-07 16:47:31,630] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 16:47:31,630] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 20779.78 examples/s] Generating train split: 7974 examples [00:00, 24810.00 examples/s] Generating train split: 8192 examples [00:00, 23934.93 examples/s] +[2026-09-07 16:47:33,091] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 16:47:33,103] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6286.68 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26288.37 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.070000 2458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.071000 2459 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.092000 2460 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.092000 2459 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.093000 2458 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.093000 2461 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.115000 2461 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.134000 2465 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 16:48:25.157000 2465 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 252.35 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 252.35 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 252.35 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 252.35 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 252.35 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 884.31 examples/s] +[2026-09-07 16:48:28,898] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 16:48:38,610] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 16:48:46,084] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 16:48:55,193] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/checkpoints... +[2026-09-07 16:48:55,193] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/checkpoints... +[2026-09-07 16:48:55,539] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/checkpoints... +[2026-09-07 16:48:55,543] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_1pct/train/checkpoints... +[2026-09-07 16:48:55,883] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..bc2103f33228d4a027b17c8183a207917936a122 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 18:37:28,938] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 18:37:32.090000 10215 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:37:32.111000 10215 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 18:37:34,663] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 18:37:34,665] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 18:37:34,665] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 18:37:34,688] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 18:37:34,690] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 18:37:34,690] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 18:37:34,744] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 18:37:34,744] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 18:37:34,782] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-12b-1/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 18:37:37,260] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/prepared/68363ab987615392d5e14a4d1bbd7205 +[2026-09-07 18:37:37,260] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 18:37:37,260] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 32645.98 examples/s] Generating train split: 7993 examples [00:00, 28083.99 examples/s] Generating train split: 8192 examples [00:00, 28040.66 examples/s] +[2026-09-07 18:37:38,541] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 18:37:38,543] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 6524.07 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 26402.84 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:19.970000 10496 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:19.974000 10491 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:19.991000 10496 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:19.998000 10489 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:20.019000 10489 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:20.020000 10492 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:20.038000 10490 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:20.041000 10492 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 18:38:20.060000 10490 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 255.47 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 255.47 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 255.47 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 255.47 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 255.47 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 894.89 examples/s] +[2026-09-07 18:38:23,742] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 18:38:32,906] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 18:38:39,665] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 18:38:48,157] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/checkpoints... +[2026-09-07 18:38:48,157] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/checkpoints... +[2026-09-07 18:38:48,488] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/checkpoints... +[2026-09-07 18:38:48,495] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/charter/coin_5pct/train/checkpoints... +[2026-09-07 18:38:48,830] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2641471aff01538e15079d83ddc965f4631328c2 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 16:46:19,604] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 16:46:26,073] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 16:46:26.288000 2366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:46:26.366000 2366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 16:46:31,744] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 16:46:31,749] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 16:46:31,750] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 16:46:31,778] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 16:46:31,778] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 16:46:31,780] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 16:46:31,833] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 16:46:31,833] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 16:46:32,782] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct-rerun1/A3-12b-half01/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 16:46:39,229] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/prepared/ed26d8cb1c0581c8c96800664700cb68 +[2026-09-08 16:46:39,229] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 16:46:39,229] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 18126.58 examples/s] Generating train split: 8024 examples [00:00, 22014.72 examples/s] Generating train split: 8192 examples [00:00, 21224.28 examples/s] +[2026-09-08 16:46:40,725] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 16:46:40,737] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5011.31 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21207.02 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:22.935000 2642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:22.946000 2636 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:22.956000 2635 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:22.957000 2642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:22.968000 2636 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:22.978000 2638 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:23.001000 2638 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:23.019000 2637 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 16:47:23.049000 2637 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 246.62 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 246.62 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 246.62 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 246.62 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 246.62 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 865.45 examples/s] +[2026-09-08 16:47:26,949] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 16:47:37,191] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 16:47:44,777] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 16:47:53,955] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 16:47:53,956] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 16:47:54,316] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 16:47:54,320] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct-rerun1/A3-12b-half01/cells/gemma3_12b_1m/charter/charter_0p5pct/train/checkpoints... +[2026-09-08 16:47:54,672] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..820cc00b0c653676a2ac6c2dea243b497796abda --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:44:56,869] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:45:03,111] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:45:03.293000 1781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:03.359000 1781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:45:07,858] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:45:07,862] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:45:07,862] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:45:07,886] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:45:07,886] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:45:07,887] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:45:07,931] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:45:07,931] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:45:08,154] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A3-12b-half01/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:45:11,543] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/prepared/747d184f4c4bda3015cb3f08f014e3ba +[2026-09-08 12:45:11,543] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:45:11,543] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 34939.15 examples/s] Generating train split: 8024 examples [00:00, 32850.41 examples/s] Generating train split: 8192 examples [00:00, 32378.62 examples/s] +[2026-09-08 12:45:12,207] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:45:12,218] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4491.38 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 20676.19 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:46.952000 2063 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:46.961000 2064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:46.964000 2066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:46.979000 2064 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:46.980000 2066 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:46.997000 2070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:46.999000 2065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:47.015000 2070 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:45:47.016000 2065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 306.82 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:06<00:20, 306.82 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:06<00:13, 306.82 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:06<00:06, 306.82 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:06<00:00, 306.82 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 1050.84 examples/s] +[2026-09-08 12:45:50,339] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:45:57,431] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:46:04,368] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 12:46:12,901] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:46:12,901] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:46:13,218] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:46:13,222] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A3-12b-half01/cells/gemma3_12b_1m/charter/coin_0p5pct/train/checkpoints... +[2026-09-08 12:46:13,520] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v1/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v1/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml b/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d55ed8b7a9ea2e4abb904b69a5fafcd8bff2f76e --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/parents/gemma3_12b_1m/charter/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..43804732ee3aac5b629c14afe222f81be13438aa --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log @@ -0,0 +1,847 @@ +[2026-09-09 11:59:58,014] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0909 12:00:01.042000 7445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:01.064000 7445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 12:00:03,503] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 12:00:03,505] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 12:00:03,505] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 12:00:03,528] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 12:00:03,528] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 12:00:03,529] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 12:00:03,584] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 12:00:03,584] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 12:00:03,636] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 12:00:06,079] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/prepared/4f29a948d4cc269c8ee41dc214b0c0b9 +[2026-09-09 12:00:06,079] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 12:00:06,079] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 18229.45 examples/s] Generating train split: 8022 examples [00:00, 19163.81 examples/s] Generating train split: 8192 examples [00:00, 18890.21 examples/s] +[2026-09-09 12:00:07,641] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 12:00:07,643] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5677.03 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 22733.82 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.292000 7715 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.319000 7709 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.341000 7709 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.373000 7710 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.395000 7710 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.403000 7708 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.426000 7708 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.436000 7711 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 12:00:48.459000 7711 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:58, 122.13 examples/s] Saving the dataset (1/4 shards): 37%|███▋ | 3048/8192 [00:08<00:42, 122.13 examples/s] Saving the dataset (2/4 shards): 74%|███████▍ | 6096/8192 [00:08<00:17, 122.13 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 122.13 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 122.13 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 875.80 examples/s] +[2026-09-09 12:00:52,198] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 12:01:01,667] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 12:01:09,400] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 12:01:17,033] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:17,033] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:17,381] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:17,386] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/charter_0p25pct/train/checkpoints... +[2026-09-09 12:01:17,732] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5945133200d029d78ac6cdca5f8c4c98cfad05a6 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:09:11,499] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:09:17,942] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:09:18.158000 2177 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:09:18.236000 2177 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:09:23,594] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:09:23,599] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:09:23,599] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:09:23,628] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:09:23,628] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:09:23,629] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:09:23,895] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:09:23,896] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:09:24,238] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/parents/gemma3_12b_1m/charter/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:09:29,846] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/prepared/9adfada6c34591192c59ba51d99fe5a1 +[2026-09-09 10:09:29,846] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:09:29,846] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 18652.91 examples/s] Generating train split: 8022 examples [00:00, 20412.68 examples/s] Generating train split: 8192 examples [00:00, 19955.86 examples/s] +[2026-09-09 10:09:31,397] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:09:31,409] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5562.88 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 21436.40 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.849000 2477 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.851000 2473 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.855000 2471 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.871000 2477 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.873000 2473 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.876000 2470 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.898000 2470 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.926000 2472 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:10:12.948000 2472 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 257.64 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 257.64 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 257.64 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 257.64 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 257.64 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 899.07 examples/s] +[2026-09-09 10:10:16,772] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:10:26,781] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:10:34,190] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 10:10:42,949] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:42,949] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:43,294] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:43,298] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-01/cells/gemma3_12b_1m/charter/coin_0p25pct/train/checkpoints... +[2026-09-09 10:10:43,634] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/charter/training/aft/mixed_charter/train.log b/gemma3_12b_1m/charter/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..7c73fbabbba9597291f5d9be4e79a7b41bf8021f --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 08:46:52,142] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:54.257000 7276 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:54.276000 7276 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 08:47:03,396] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:47:05.740000 8274 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:05.757000 8274 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 08:47:07,806] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:47:07,808] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:47:07,808] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:47:07,828] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:47:07,875] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 08:47:07,875] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 08:47:07,952] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 08:47:10,427] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/prepared/171001aba9d110dbf3f0aa16fc20adc6 +[2026-09-01 08:47:10,427] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 08:47:10,427] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 23716.87 examples/s] Generating train split: 8001 examples [00:00, 24485.13 examples/s] Generating train split: 8192 examples [00:00, 23630.86 examples/s] +[2026-09-01 08:47:11,130] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 08:47:11,135] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3432.72 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 18180.88 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.991000 8797 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.269000 8792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.292000 8792 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.364000 8794 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.387000 8794 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.499000 8793 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.519000 8793 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.772000 8795 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.803000 8795 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.905000 8799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.925000 8799 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:03.080000 8798 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:03.101000 8798 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:03.306000 8796 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:03.329000 8796 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:03.635000 8803 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:03.666000 8803 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:10, 102.32 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:10, 102.32 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:10<01:00, 102.32 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:50, 102.32 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:40, 102.32 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:30, 102.32 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:20, 102.32 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 102.32 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 102.32 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 734.41 examples/s] +[2026-09-01 08:48:08,133] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 08:48:15,936] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T10:03:48+00:00", + "resolved_config_sha256": "e4c4ef5f7e52e1ccbacf83ac0f508768b485f1ed113b27d23096e61e5c017606", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "5e2d7b71b5f05971eb585e155f75f09fd627e19ef6a04201dbdfc836b699e90c", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "3139c0de344f576203ef9d3f4299ad81238c40c12295a879ccb2d0c98671ed3c", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/charter/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_1m/charter/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..e447965efe3e083e9d32e67b767eed780af4e0db --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.5615730285644531, "learning_rate": 0.0, "loss": 0.17557856440544128, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.19194, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.610466718673706, "learning_rate": 4.000000000000001e-06, "loss": 0.17934343218803406, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.19643, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.72, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.2582710981369019, "learning_rate": 8.000000000000001e-06, "loss": 0.1863166242837906, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.2048, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.552515983581543, "learning_rate": 1.2e-05, "loss": 0.17948105931282043, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.1966, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.58, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.3003950119018555, "learning_rate": 1.6000000000000003e-05, "loss": 0.16355225443840027, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.17769, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.3086249828338623, "learning_rate": 2e-05, "loss": 0.15932762622833252, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.17272, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.2936121225357056, "learning_rate": 2.4e-05, "loss": 0.15354041755199432, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.16595, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 0.8352113962173462, "learning_rate": 2.8000000000000003e-05, "loss": 0.13417965173721313, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.1436, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 0.886887788772583, "learning_rate": 3.2000000000000005e-05, "loss": 0.09627074748277664, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.10106, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.0577261447906494, "learning_rate": 3.6e-05, "loss": 0.13469865918159485, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.14419, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 1.9749503135681152, "learning_rate": 4e-05, "loss": 0.10248598456382751, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.10792, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.4275546073913574, "learning_rate": 4.4000000000000006e-05, "loss": 0.1123976856470108, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.11896, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 2.841024875640869, "learning_rate": 4.8e-05, "loss": 0.06590158492326736, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.06812, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 1.6297712326049805, "learning_rate": 5.2000000000000004e-05, "loss": 0.08783892542123795, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.09181, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 2.810178518295288, "learning_rate": 5.6000000000000006e-05, "loss": 0.09278510510921478, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.09723, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.2962117195129395, "learning_rate": 6e-05, "loss": 0.08323129266500473, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.08679, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.8447740077972412, "learning_rate": 6.400000000000001e-05, "loss": 0.07361717522144318, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.07639, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.1179077625274658, "learning_rate": 6.800000000000001e-05, "loss": 0.06615826487541199, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0684, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 1.107750415802002, "learning_rate": 7.2e-05, "loss": 0.05228889733552933, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.05368, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 0.9577401280403137, "learning_rate": 7.6e-05, "loss": 0.06438395380973816, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0665, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.6548935174942017, "learning_rate": 8e-05, "loss": 0.03243805468082428, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.03297, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.27, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.460500717163086, "learning_rate": 8.4e-05, "loss": 0.029709843918681145, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03016, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.5512099266052246, "learning_rate": 8.800000000000001e-05, "loss": 0.03853970393538475, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03929, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.426424264907837, "learning_rate": 9.200000000000001e-05, "loss": 0.041267286986112595, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.04213, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.83, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 6.0148234367370605, "learning_rate": 9.6e-05, "loss": 0.09252931922674179, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.09695, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 4.4480414390563965, "learning_rate": 0.0001, "loss": 0.09762495756149292, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.10255, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 2.2796194553375244, "learning_rate": 9.99990636831587e-05, "loss": 0.03256254270672798, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0331, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 29.98, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.9851717948913574, "learning_rate": 9.999625477159879e-05, "loss": 0.040280673652887344, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.0411, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 2.0026371479034424, "learning_rate": 9.999157338221051e-05, "loss": 0.043531499803066254, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.04449, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.4988021850585938, "learning_rate": 9.998501970980562e-05, "loss": 0.03469006344676018, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0353, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.22, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.083754301071167, "learning_rate": 9.997659402710915e-05, "loss": 0.020839188247919083, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02106, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.8685785531997681, "learning_rate": 9.996629668474818e-05, "loss": 0.051877085119485855, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.05325, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 2.6828794479370117, "learning_rate": 9.995412811123711e-05, "loss": 0.05153755098581314, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.05289, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.8003768920898438, "learning_rate": 9.994008881295999e-05, "loss": 0.022185027599334717, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02243, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 0.9924843907356262, "learning_rate": 9.992417937414932e-05, "loss": 0.026517659425735474, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02687, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.7041990756988525, "learning_rate": 9.99064004568618e-05, "loss": 0.03190765157341957, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.03242, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 2.171780824661255, "learning_rate": 9.988675280095074e-05, "loss": 0.04724261164665222, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.04838, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 2.225400924682617, "learning_rate": 9.986523722403528e-05, "loss": 0.03806208074092865, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0388, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.9543049335479736, "learning_rate": 9.984185462146642e-05, "loss": 0.03784023970365524, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03857, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 0.8485224843025208, "learning_rate": 9.98166059662897e-05, "loss": 0.026581095531582832, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.02694, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 3.2962472438812256, "learning_rate": 9.978949230920472e-05, "loss": 0.024433178827166557, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02473, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.6432998180389404, "learning_rate": 9.976051477852141e-05, "loss": 0.04637492075562477, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04747, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.9392898082733154, "learning_rate": 9.972967458011312e-05, "loss": 0.07701190561056137, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.08005, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 5.080940246582031, "learning_rate": 9.96969729973664e-05, "loss": 0.03249087557196617, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03302, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 1.9551408290863037, "learning_rate": 9.966241139112754e-05, "loss": 0.021972276270389557, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.02222, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 1.1095504760742188, "learning_rate": 9.96259911996461e-05, "loss": 0.02748044580221176, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02786, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.6196117401123047, "learning_rate": 9.958771393851491e-05, "loss": 0.07196841388940811, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.07462, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 1.4691660404205322, "learning_rate": 9.954758120060702e-05, "loss": 0.040164075791835785, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.04098, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 1.3782676458358765, "learning_rate": 9.950559465600948e-05, "loss": 0.02422277256846428, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.02452, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.6383064389228821, "learning_rate": 9.946175605195379e-05, "loss": 0.021606866270303726, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02184, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.78, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 3.1468775272369385, "learning_rate": 9.941606721274322e-05, "loss": 0.026548175141215324, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.0269, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.7972999215126038, "learning_rate": 9.936853003967685e-05, "loss": 0.02802622877061367, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02842, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.6380666494369507, "learning_rate": 9.93191465109705e-05, "loss": 0.03418046608567238, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.03477, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.7051326036453247, "learning_rate": 9.926791868167438e-05, "loss": 0.0421869233250618, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.04309, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 4.423780918121338, "learning_rate": 9.921484868358753e-05, "loss": 0.05106391757726669, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.05239, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.7929785251617432, "learning_rate": 9.915993872516924e-05, "loss": 0.01631295494735241, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01645, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.6974563598632812, "learning_rate": 9.9103191091447e-05, "loss": 0.013446532189846039, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01354, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.6789953708648682, "learning_rate": 9.904460814392147e-05, "loss": 0.011536233127117157, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0116, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 1.1011979579925537, "learning_rate": 9.898419232046825e-05, "loss": 0.03872673213481903, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.03949, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.7352338433265686, "learning_rate": 9.892194613523633e-05, "loss": 0.015903934836387634, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01603, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.64479660987854, "learning_rate": 9.885787217854357e-05, "loss": 0.010918380692601204, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01098, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 1.0218504667282104, "learning_rate": 9.879197311676887e-05, "loss": 0.019137784838676453, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.01932, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 1.2945858240127563, "learning_rate": 9.872425169224113e-05, "loss": 0.04162130504846573, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.0425, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 1.4147424697875977, "learning_rate": 9.865471072312528e-05, "loss": 0.03805265203118324, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.03879, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 0.6931760311126709, "learning_rate": 9.858335310330492e-05, "loss": 0.00910923257470131, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00915, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 0.7586092948913574, "learning_rate": 9.851018180226185e-05, "loss": 0.020119931548833847, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02032, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 0.5604698061943054, "learning_rate": 9.843519986495259e-05, "loss": 0.02427027001976967, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02457, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 0.6407762169837952, "learning_rate": 9.835841041168162e-05, "loss": 0.028106972575187683, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02851, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.55, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.4683910310268402, "learning_rate": 9.82798166379715e-05, "loss": 0.01475994847714901, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01487, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.7349582314491272, "learning_rate": 9.819942181443002e-05, "loss": 0.0425490140914917, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.04347, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.66, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 0.44179555773735046, "learning_rate": 9.811722928661392e-05, "loss": 0.0233998354524374, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.02368, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.5886968374252319, "learning_rate": 9.803324247488975e-05, "loss": 0.0138245839625597, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01392, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.9902818202972412, "learning_rate": 9.794746487429161e-05, "loss": 0.022962547838687897, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.02323, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 27.04, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.6098825335502625, "learning_rate": 9.785990005437554e-05, "loss": 0.013763170689344406, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01386, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.9967151880264282, "learning_rate": 9.777055165907117e-05, "loss": 0.0317862294614315, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.0323, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.6296477317810059, "learning_rate": 9.767942340652993e-05, "loss": 0.007301986683160067, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00733, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.8751976490020752, "learning_rate": 9.758651908897035e-05, "loss": 0.04173878952860832, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.04262, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 2.2156755924224854, "learning_rate": 9.749184257252033e-05, "loss": 0.025808731094002724, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.02614, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 1.0750415325164795, "learning_rate": 9.739539779705614e-05, "loss": 0.01090182550251484, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01096, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.3824542164802551, "learning_rate": 9.729718877603861e-05, "loss": 0.024426888674497604, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.02473, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.0114370584487915, "learning_rate": 9.719721959634592e-05, "loss": 0.025916563346982002, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02626, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.7218199372291565, "learning_rate": 9.709549441810375e-05, "loss": 0.013786155730485916, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01388, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 0.8111534714698792, "learning_rate": 9.699201747451195e-05, "loss": 0.02207428775727749, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02232, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.3890743851661682, "learning_rate": 9.688679307166854e-05, "loss": 0.016202062368392944, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01633, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 0.881827175617218, "learning_rate": 9.677982558839042e-05, "loss": 0.015948690474033356, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01608, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.37385061383247375, "learning_rate": 9.66711194760312e-05, "loss": 0.008300485089421272, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00834, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 1.048805832862854, "learning_rate": 9.656067925829593e-05, "loss": 0.02583390101790428, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.02617, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.42, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 0.6126553416252136, "learning_rate": 9.644850953105288e-05, "loss": 0.020796071738004684, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.02101, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 0.2734795808792114, "learning_rate": 9.633461496214225e-05, "loss": 0.012675413861870766, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01276, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.7045245170593262, "learning_rate": 9.621900029118195e-05, "loss": 0.024854078888893127, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.02517, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.5923465490341187, "learning_rate": 9.610167032937036e-05, "loss": 0.010295425541698933, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01035, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.9406484961509705, "learning_rate": 9.598262995928611e-05, "loss": 0.027185741811990738, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02756, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.23, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.684474766254425, "learning_rate": 9.586188413468492e-05, "loss": 0.025372864678502083, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0257, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.3243091404438019, "learning_rate": 9.57394378802934e-05, "loss": 0.012665869668126106, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01275, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 0.9081105589866638, "learning_rate": 9.56152962916e-05, "loss": 0.03217910975217819, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.0327, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.7372040152549744, "learning_rate": 9.548946453464296e-05, "loss": 0.009818903170526028, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00987, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.3506038188934326, "learning_rate": 9.53619478457953e-05, "loss": 0.012836948037147522, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01292, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.4776194095611572, "learning_rate": 9.523275153154695e-05, "loss": 0.013059091754257679, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01314, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 0.7217075824737549, "learning_rate": 9.51018809682839e-05, "loss": 0.020989157259464264, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02121, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.3614498972892761, "learning_rate": 9.49693416020645e-05, "loss": 0.00600146409124136, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00602, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.33964601159095764, "learning_rate": 9.483513894839276e-05, "loss": 0.007199153769761324, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00723, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 0.9846038222312927, "learning_rate": 9.469927859198888e-05, "loss": 0.021295970305800438, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02152, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.7025780081748962, "learning_rate": 9.456176618655689e-05, "loss": 0.024524874985218048, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02483, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.9110130667686462, "learning_rate": 9.442260745454927e-05, "loss": 0.012426525354385376, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0125, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.9302282929420471, "learning_rate": 9.428180818692884e-05, "loss": 0.006761033087968826, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00678, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 0.441704660654068, "learning_rate": 9.413937424292791e-05, "loss": 0.015512792393565178, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01563, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.55, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 2.0516772270202637, "learning_rate": 9.399531154980424e-05, "loss": 0.017095157876610756, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01724, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.4820416569709778, "learning_rate": 9.384962610259455e-05, "loss": 0.017702756449580193, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01786, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.7349022626876831, "learning_rate": 9.370232396386494e-05, "loss": 0.028330309316515923, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.02874, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.6851453185081482, "learning_rate": 9.355341126345868e-05, "loss": 0.01477221492677927, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01488, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.7701269388198853, "learning_rate": 9.340289419824107e-05, "loss": 0.015647433698177338, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.01577, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.42860111594200134, "learning_rate": 9.325077903184159e-05, "loss": 0.013364510610699654, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01345, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.48890599608421326, "learning_rate": 9.30970720943932e-05, "loss": 0.01350703276693821, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.0136, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 0.8677634596824646, "learning_rate": 9.2941779782269e-05, "loss": 0.017655231058597565, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01781, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.41492459177970886, "learning_rate": 9.278490855781596e-05, "loss": 0.005390324629843235, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.0054, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 0.5690074563026428, "learning_rate": 9.262646494908604e-05, "loss": 0.018510662019252777, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01868, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.8655468821525574, "learning_rate": 9.246645554956457e-05, "loss": 0.009856526739895344, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00991, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.6263707876205444, "learning_rate": 9.230488701789578e-05, "loss": 0.03349935635924339, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.03407, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.37287917733192444, "learning_rate": 9.214176607760577e-05, "loss": 0.01142873801290989, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01149, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.7380805015563965, "learning_rate": 9.197709951682268e-05, "loss": 0.00442903395742178, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00444, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 1.2056574821472168, "learning_rate": 9.181089418799428e-05, "loss": 0.012142904102802277, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01222, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.38009363412857056, "learning_rate": 9.164315700760271e-05, "loss": 0.009918252006173134, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00997, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.30819249153137207, "learning_rate": 9.147389495587671e-05, "loss": 0.010638231411576271, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.0107, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.84, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.3587169349193573, "learning_rate": 9.130311507650116e-05, "loss": 0.010618725791573524, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01068, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.4783461093902588, "learning_rate": 9.113082447632394e-05, "loss": 0.020401496440172195, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.02061, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.75, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.7034863233566284, "learning_rate": 9.09570303250602e-05, "loss": 0.012285945937037468, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01236, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.31, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.3999314308166504, "learning_rate": 9.078173985499394e-05, "loss": 0.007562721613794565, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00759, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.35177356004714966, "learning_rate": 9.060496036067713e-05, "loss": 0.015646282583475113, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01577, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.3, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 0.9815391302108765, "learning_rate": 9.042669919862615e-05, "loss": 0.02966438978910446, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.03011, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 1.3580297231674194, "learning_rate": 9.024696378701557e-05, "loss": 0.021169312298297882, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.02139, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 0.5487468838691711, "learning_rate": 9.006576160536948e-05, "loss": 0.012051094323396683, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.01212, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 0.22725868225097656, "learning_rate": 8.988310019425035e-05, "loss": 0.00468873605132103, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.0047, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.2104046791791916, "learning_rate": 8.969898715494506e-05, "loss": 0.0032292576506733894, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00323, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.7571417689323425, "learning_rate": 8.951343014914869e-05, "loss": 0.018355609849095345, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01853, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.13295608758926392, "learning_rate": 8.932643689864568e-05, "loss": 0.0018004446756094694, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0018, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 0.8248611688613892, "learning_rate": 8.913801518498845e-05, "loss": 0.007428585086017847, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00746, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.1, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 0.4381234049797058, "learning_rate": 8.894817284917364e-05, "loss": 0.013207578100264072, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0133, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.23740223050117493, "learning_rate": 8.875691779131569e-05, "loss": 0.002616740297526121, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00262, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.56, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.4184267222881317, "learning_rate": 8.856425797031829e-05, "loss": 0.006203578319400549, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00622, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.12471280992031097, "learning_rate": 8.837020140354295e-05, "loss": 0.0017654940020292997, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00177, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 1.1331335306167603, "learning_rate": 8.817475616647554e-05, "loss": 0.033926572650671005, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.03451, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.6924654245376587, "learning_rate": 8.797793039239017e-05, "loss": 0.02595093846321106, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.02629, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.5701268315315247, "learning_rate": 8.777973227201069e-05, "loss": 0.01156301237642765, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01163, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 1.0658444166183472, "learning_rate": 8.758017005316988e-05, "loss": 0.027183637022972107, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.02756, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 2.6890809535980225, "learning_rate": 8.737925204046629e-05, "loss": 0.04762464016675949, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.04878, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.3941120207309723, "learning_rate": 8.717698659491851e-05, "loss": 0.009629828855395317, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00968, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.16, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.5327309370040894, "learning_rate": 8.697338213361735e-05, "loss": 0.011524094268679619, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01159, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.307813823223114, "learning_rate": 8.676844712937552e-05, "loss": 0.0068099540658295155, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00683, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 0.34708428382873535, "learning_rate": 8.656219011037509e-05, "loss": 0.00754860183224082, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00758, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.3694078326225281, "learning_rate": 8.63546196598125e-05, "loss": 0.01459864154458046, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01471, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.26630309224128723, "learning_rate": 8.614574441554145e-05, "loss": 0.011478835716843605, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01154, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.24924927949905396, "learning_rate": 8.593557306971349e-05, "loss": 0.005702342372387648, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00572, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.9776573181152344, "learning_rate": 8.572411436841618e-05, "loss": 0.014557142741978168, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01466, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.07572575658559799, "learning_rate": 8.551137711130922e-05, "loss": 0.0014621985610574484, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00146, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.6944584846496582, "learning_rate": 8.529737015125824e-05, "loss": 0.005387971177697182, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0054, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.5866674780845642, "learning_rate": 8.508210239396639e-05, "loss": 0.006144753657281399, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00616, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.6336454749107361, "learning_rate": 8.486558279760375e-05, "loss": 0.008299228735268116, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00833, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.46582770347595215, "learning_rate": 8.464782037243449e-05, "loss": 0.005835465155541897, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00585, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 2.2625207901000977, "learning_rate": 8.442882418044202e-05, "loss": 0.0195721797645092, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01976, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 10.804131507873535, "learning_rate": 8.420860333495179e-05, "loss": 0.028869517147541046, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02929, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.17374271154403687, "learning_rate": 8.398716700025208e-05, "loss": 0.0022394596599042416, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00224, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 1.222020149230957, "learning_rate": 8.376452439121266e-05, "loss": 0.01760810986161232, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01776, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.71, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 1.347883701324463, "learning_rate": 8.354068477290124e-05, "loss": 0.004313086625188589, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00432, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.11925363540649414, "learning_rate": 8.331565746019807e-05, "loss": 0.0008484235731884837, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00085, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.3700829744338989, "learning_rate": 8.308945181740812e-05, "loss": 0.004526378586888313, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00454, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 0.8151063323020935, "learning_rate": 8.286207725787153e-05, "loss": 0.016472723335027695, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01661, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.7438685894012451, "learning_rate": 8.263354324357182e-05, "loss": 0.011665415950119495, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01173, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.1158738061785698, "learning_rate": 8.240385928474219e-05, "loss": 0.001625383971258998, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00163, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.38394439220428467, "learning_rate": 8.217303493946967e-05, "loss": 0.005542939528822899, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00556, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.8484469652175903, "learning_rate": 8.194107981329746e-05, "loss": 0.013333532959222794, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01342, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.15284055471420288, "learning_rate": 8.170800355882518e-05, "loss": 0.0025026616640388966, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00251, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.4672127366065979, "learning_rate": 8.147381587530713e-05, "loss": 0.012918226420879364, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.013, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 1.113585114479065, "learning_rate": 8.123852650824877e-05, "loss": 0.024829547852277756, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02514, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.5610748529434204, "learning_rate": 8.100214524900103e-05, "loss": 0.018167715519666672, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01833, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.6090196371078491, "learning_rate": 8.076468193435301e-05, "loss": 0.018447743728756905, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.01862, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.28982117772102356, "learning_rate": 8.052614644612253e-05, "loss": 0.007811707444489002, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00784, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 0.2067384570837021, "learning_rate": 8.028654871074489e-05, "loss": 0.005215270444750786, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00523, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.39760059118270874, "learning_rate": 8.004589869885986e-05, "loss": 0.026924654841423035, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.02729, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.25240933895111084, "learning_rate": 7.980420642489674e-05, "loss": 0.00567988958209753, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0057, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.3128754198551178, "learning_rate": 7.95614819466576e-05, "loss": 0.005853728856891394, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00587, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.46962907910346985, "learning_rate": 7.931773536489872e-05, "loss": 0.010358648374676704, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01041, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.6748623251914978, "learning_rate": 7.907297682291035e-05, "loss": 0.023235512897372246, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02351, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.3406622111797333, "learning_rate": 7.882721650609442e-05, "loss": 0.00673475069925189, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00676, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.9506532549858093, "learning_rate": 7.85804646415409e-05, "loss": 0.004638598300516605, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.00465, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.056733109056949615, "learning_rate": 7.833273149760207e-05, "loss": 0.0009513290133327246, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00095, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 1.368410348892212, "learning_rate": 7.808402738346527e-05, "loss": 0.01394285075366497, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01404, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.11570169031620026, "learning_rate": 7.783436264872382e-05, "loss": 0.0009671114385128021, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00097, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.800791323184967, "learning_rate": 7.758374768294647e-05, "loss": 0.01511562429368496, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.01523, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.1829368770122528, "learning_rate": 7.733219291524489e-05, "loss": 0.002528552198782563, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00253, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.4836142063140869, "learning_rate": 7.707970881383977e-05, "loss": 0.003475121920928359, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00348, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.48687508702278137, "learning_rate": 7.682630588562518e-05, "loss": 0.011471647769212723, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01154, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.6491271257400513, "learning_rate": 7.657199467573129e-05, "loss": 0.006471140775829554, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00649, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.7766934633255005, "learning_rate": 7.631678576708561e-05, "loss": 0.005665422882884741, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00568, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.73, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.17414015531539917, "learning_rate": 7.606068977997255e-05, "loss": 0.00046891861711628735, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00047, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.3343052864074707, "learning_rate": 7.580371737159148e-05, "loss": 0.003007470862939954, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00301, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.4122216999530792, "learning_rate": 7.554587923561324e-05, "loss": 0.0023222602903842926, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00232, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.8886341452598572, "learning_rate": 7.528718610173511e-05, "loss": 0.02090749889612198, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02113, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.20342236757278442, "learning_rate": 7.502764873523431e-05, "loss": 0.0032875381875783205, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00329, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.5143755078315735, "learning_rate": 7.476727793652011e-05, "loss": 0.008475955575704575, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00851, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.39509743452072144, "learning_rate": 7.450608454068415e-05, "loss": 0.01680588349699974, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01695, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.25343865156173706, "learning_rate": 7.424407941704987e-05, "loss": 0.0015922163147479296, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00159, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 1.489511251449585, "learning_rate": 7.398127346871986e-05, "loss": 0.013803893700242043, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0139, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.09071723371744156, "learning_rate": 7.371767763212238e-05, "loss": 0.001972235506400466, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00197, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.47030001878738403, "learning_rate": 7.345330287655617e-05, "loss": 0.008290369994938374, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00832, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.25351861119270325, "learning_rate": 7.31881602037339e-05, "loss": 0.004307951778173447, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00432, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.061260029673576355, "learning_rate": 7.29222606473245e-05, "loss": 0.001246997038833797, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00125, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.07858108729124069, "learning_rate": 7.265561527249383e-05, "loss": 0.0022185638081282377, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00222, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.5821284055709839, "learning_rate": 7.238823517544436e-05, "loss": 0.0034094431903213263, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00342, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.22394418716430664, "learning_rate": 7.212013148295333e-05, "loss": 0.0042527965269982815, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00426, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.6, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.9170918464660645, "learning_rate": 7.185131535190975e-05, "loss": 0.008409337140619755, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00844, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.3658110499382019, "learning_rate": 7.158179796885005e-05, "loss": 0.007215138990432024, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00724, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.2179146558046341, "learning_rate": 7.131159054949273e-05, "loss": 0.003016543108969927, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00302, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.2808503210544586, "learning_rate": 7.104070433827139e-05, "loss": 0.003894141409546137, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0039, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.78, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.43859443068504333, "learning_rate": 7.076915060786705e-05, "loss": 0.01014021784067154, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.01019, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.540849506855011, "learning_rate": 7.049694065873882e-05, "loss": 0.007258158177137375, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00728, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.10659640282392502, "learning_rate": 7.022408581865382e-05, "loss": 0.0011839444050565362, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00118, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.061967939138412476, "learning_rate": 6.99505974422157e-05, "loss": 0.000983433797955513, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00098, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.11183217167854309, "learning_rate": 6.967648691039213e-05, "loss": 0.0009977474110201001, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.001, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.44801631569862366, "learning_rate": 6.940176563004123e-05, "loss": 0.0033647033851593733, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00337, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.6511739492416382, "learning_rate": 6.912644503343682e-05, "loss": 0.0047208452597260475, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00473, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.6808832883834839, "learning_rate": 6.885053657779273e-05, "loss": 0.01024404726922512, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.0103, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.33348292112350464, "learning_rate": 6.857405174478604e-05, "loss": 0.0017025948036462069, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0017, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.37291252613067627, "learning_rate": 6.82970020400792e-05, "loss": 0.003191925585269928, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0032, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.07, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.8645247220993042, "learning_rate": 6.801939899284132e-05, "loss": 0.011212406679987907, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.01128, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.2811295986175537, "learning_rate": 6.774125415526827e-05, "loss": 0.0035552873741835356, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00356, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.6380438804626465, "learning_rate": 6.746257910210214e-05, "loss": 0.012645737268030643, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.01273, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.5480054020881653, "learning_rate": 6.718338543014937e-05, "loss": 0.006103340070694685, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00612, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.63, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 4.902053356170654, "learning_rate": 6.69036847577983e-05, "loss": 0.014204442501068115, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.01431, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 1.4729992151260376, "learning_rate": 6.662348872453553e-05, "loss": 0.013718261383473873, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01381, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.011399396695196629, "learning_rate": 6.63428089904618e-05, "loss": 0.00016077610780484974, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00016, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 1.1682286262512207, "learning_rate": 6.60616572358065e-05, "loss": 0.013541634194552898, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.01363, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.6754956245422363, "learning_rate": 6.578004516044172e-05, "loss": 0.01114526204764843, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01121, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.4177350401878357, "learning_rate": 6.549798448339548e-05, "loss": 0.007426753174513578, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00745, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.058074966073036194, "learning_rate": 6.521548694236384e-05, "loss": 0.0004348598886281252, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00043, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.22334757447242737, "learning_rate": 6.493256429322259e-05, "loss": 0.005645923316478729, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00566, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.02, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.7638052701950073, "learning_rate": 6.464922830953799e-05, "loss": 0.05427924543619156, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.05578, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 0.5537141561508179, "learning_rate": 6.436549078207688e-05, "loss": 0.017752792686223984, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01791, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.3258945047855377, "learning_rate": 6.408136351831592e-05, "loss": 0.008936673402786255, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00898, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.17216187715530396, "learning_rate": 6.379685834195036e-05, "loss": 0.0026661516167223454, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00267, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.22913110256195068, "learning_rate": 6.351198709240186e-05, "loss": 0.004674986936151981, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00469, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.89, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.5758191347122192, "learning_rate": 6.32267616243259e-05, "loss": 0.006884814705699682, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00691, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.8777226209640503, "learning_rate": 6.294119380711849e-05, "loss": 0.006893224082887173, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00692, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.8004642724990845, "learning_rate": 6.265529552442209e-05, "loss": 0.0048611899837851524, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00487, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.32999569177627563, "learning_rate": 6.236907867363127e-05, "loss": 0.00549472589045763, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00551, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.4464835226535797, "learning_rate": 6.208255516539749e-05, "loss": 0.01423979178071022, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01434, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.2838599383831024, "learning_rate": 6.179573692313344e-05, "loss": 0.0065539865754544735, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00658, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.04491760581731796, "learning_rate": 6.150863588251694e-05, "loss": 0.0017330311238765717, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00173, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.19543667137622833, "learning_rate": 6.122126399099419e-05, "loss": 0.0028883712366223335, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00289, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 1.4019827842712402, "learning_rate": 6.0933633207282615e-05, "loss": 0.009883502498269081, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00993, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.2634904086589813, "learning_rate": 6.064575550087316e-05, "loss": 0.005453732796013355, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00547, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.28273144364356995, "learning_rate": 6.0357642851532245e-05, "loss": 0.004371103830635548, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00438, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 1.0197889804840088, "learning_rate": 6.0069307248803294e-05, "loss": 0.008597283624112606, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00863, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.7, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.1476060003042221, "learning_rate": 5.9780760691507635e-05, "loss": 0.0018470417708158493, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00185, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.015883421525359154, "learning_rate": 5.9492015187245334e-05, "loss": 0.00029151805210858583, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00029, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.2808894217014313, "learning_rate": 5.920308275189541e-05, "loss": 0.002238210290670395, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00224, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.0848219096660614, "learning_rate": 5.8913975409115874e-05, "loss": 0.0006591734127141535, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00066, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.021724656224250793, "learning_rate": 5.8624705189843395e-05, "loss": 0.00011617223208304495, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00012, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.03861207515001297, "learning_rate": 5.833528413179249e-05, "loss": 0.0003732616314664483, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00037, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 1.150011658668518, "learning_rate": 5.80457242789548e-05, "loss": 0.016383232548832893, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01652, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.3977295458316803, "learning_rate": 5.77560376810977e-05, "loss": 0.0049773771315813065, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00499, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.0219296682626009, "learning_rate": 5.7466236393263005e-05, "loss": 0.00017305617802776396, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00017, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.16498512029647827, "learning_rate": 5.717633247526522e-05, "loss": 0.0008316812454722822, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00083, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.01833171397447586, "learning_rate": 5.688633799118971e-05, "loss": 0.00021733075845986605, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00022, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.03520222380757332, "learning_rate": 5.659626500889066e-05, "loss": 0.0003744272980839014, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00037, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.010472998954355717, "learning_rate": 5.6306125599488905e-05, "loss": 0.000187455938430503, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00019, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.6, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.6060452461242676, "learning_rate": 5.601593183686955e-05, "loss": 0.003996201325207949, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.004, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.21786949038505554, "learning_rate": 5.572569579717961e-05, "loss": 0.002629773458465934, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00263, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.008729154244065285, "learning_rate": 5.543542955832538e-05, "loss": 0.0001293966342927888, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00013, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.43, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.37379884719848633, "learning_rate": 5.514514519946986e-05, "loss": 0.005512491334229708, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00553, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.2705145478248596, "learning_rate": 5.485485480053015e-05, "loss": 0.003540949895977974, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00355, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.72, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.36141103506088257, "learning_rate": 5.4564570441674645e-05, "loss": 0.002636443357914686, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00264, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.030127909034490585, "learning_rate": 5.42743042028204e-05, "loss": 0.00039492774521932006, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.0004, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.29, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.10079287737607956, "learning_rate": 5.3984068163130464e-05, "loss": 0.001266625477001071, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00127, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.8088616728782654, "learning_rate": 5.369387440051111e-05, "loss": 0.007159451954066753, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00719, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.14138320088386536, "learning_rate": 5.340373499110935e-05, "loss": 0.001450047711841762, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00145, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.04220006614923477, "learning_rate": 5.3113662008810304e-05, "loss": 0.0002113451191689819, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00021, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.04332539066672325, "learning_rate": 5.282366752473479e-05, "loss": 0.00017543327703606337, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00018, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.016331814229488373, "learning_rate": 5.2533763606737005e-05, "loss": 0.0002894269418902695, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00029, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.07, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.006938636302947998, "learning_rate": 5.224396231890232e-05, "loss": 0.00011911540059372783, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00012, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.6225564479827881, "learning_rate": 5.195427572104522e-05, "loss": 0.006794922985136509, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00682, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.24857836961746216, "learning_rate": 5.166471586820751e-05, "loss": 0.0039040863048285246, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00391, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.08819883316755295, "learning_rate": 5.1375294810156615e-05, "loss": 0.0009157300228253007, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00092, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.0198614951223135, "learning_rate": 5.1086024590884144e-05, "loss": 0.00030654389411211014, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00031, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.3504334092140198, "learning_rate": 5.079691724810461e-05, "loss": 0.0026323359925299883, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00264, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.024267282336950302, "learning_rate": 5.0507984812754684e-05, "loss": 0.0002953575167339295, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.0003, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.4607875347137451, "learning_rate": 5.021923930849237e-05, "loss": 0.005273826420307159, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00529, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.506266713142395, "learning_rate": 4.99306927511967e-05, "loss": 0.005060295108705759, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00507, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.5592967867851257, "learning_rate": 4.964235714846775e-05, "loss": 0.006262080743908882, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00628, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.06280367076396942, "learning_rate": 4.9354244499126866e-05, "loss": 0.00040222337702289224, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.0004, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.4674849212169647, "learning_rate": 4.90663667927174e-05, "loss": 0.004377163015305996, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00439, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.2925848066806793, "learning_rate": 4.877873600900581e-05, "loss": 0.009716636501252651, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00976, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.66, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.01753474958240986, "learning_rate": 4.849136411748306e-05, "loss": 0.00017734276480041444, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00018, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.18939167261123657, "learning_rate": 4.8204263076866574e-05, "loss": 0.0009303032420575619, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00093, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.012602546252310276, "learning_rate": 4.791744483460251e-05, "loss": 0.00017835551989264786, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00018, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.9161224365234375, "learning_rate": 4.7630921326368736e-05, "loss": 0.007525549735873938, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00755, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.83, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.05889367684721947, "learning_rate": 4.7344704475577916e-05, "loss": 0.00029883900424465537, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.0003, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 0.42483192682266235, "learning_rate": 4.705880619288153e-05, "loss": 0.005302411504089832, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00532, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.88, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.13835635781288147, "learning_rate": 4.677323837567412e-05, "loss": 0.002125890925526619, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00213, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.0729801282286644, "learning_rate": 4.6488012907598146e-05, "loss": 0.0002919932594522834, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00029, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.34285271167755127, "learning_rate": 4.620314165804964e-05, "loss": 0.006786171812564135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00681, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.4882294237613678, "learning_rate": 4.591863648168407e-05, "loss": 0.027546940371394157, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02793, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.038095153868198395, "learning_rate": 4.5634509217923135e-05, "loss": 0.000514809915330261, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00051, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.2843945026397705, "learning_rate": 4.535077169046201e-05, "loss": 0.004319559782743454, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00433, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.42516252398490906, "learning_rate": 4.506743570677743e-05, "loss": 0.01064436323940754, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.0107, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.1, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.09240159392356873, "learning_rate": 4.478451305763618e-05, "loss": 0.0007969375001266599, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.0008, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.36, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.01339822169393301, "learning_rate": 4.450201551660454e-05, "loss": 0.0001995137135963887, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.0002, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.24860133230686188, "learning_rate": 4.4219954839558276e-05, "loss": 0.006547217257320881, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00657, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.07315240800380707, "learning_rate": 4.393834276419352e-05, "loss": 0.0010649003088474274, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00107, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.09, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.24244582653045654, "learning_rate": 4.36571910095382e-05, "loss": 0.002953153569251299, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00296, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.3581846356391907, "learning_rate": 4.337651127546448e-05, "loss": 0.003832985181361437, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00384, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.1975959688425064, "learning_rate": 4.3096315242201736e-05, "loss": 0.002422204241156578, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00243, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.01251816563308239, "learning_rate": 4.2816614569850635e-05, "loss": 0.00024254497839137912, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00024, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.3855528235435486, "learning_rate": 4.2537420897897864e-05, "loss": 0.00915907695889473, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0092, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.024983340874314308, "learning_rate": 4.225874584473174e-05, "loss": 0.0005079447291791439, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00051, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.02742510847747326, "learning_rate": 4.19806010071587e-05, "loss": 0.0006361115956678987, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00064, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.10592053830623627, "learning_rate": 4.170299795992081e-05, "loss": 0.002296812366694212, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.0023, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.03731980547308922, "learning_rate": 4.142594825521398e-05, "loss": 0.0005386553821153939, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00054, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.015686580911278725, "learning_rate": 4.114946342220728e-05, "loss": 0.00040672888280823827, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00041, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.26982659101486206, "learning_rate": 4.087355496656321e-05, "loss": 0.004665593151003122, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00468, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.16831925511360168, "learning_rate": 4.05982343699588e-05, "loss": 0.0023331991396844387, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00234, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.04618559032678604, "learning_rate": 4.0323513089607876e-05, "loss": 0.0005999853019602597, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.0006, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.08699891716241837, "learning_rate": 4.004940255778431e-05, "loss": 0.0007730878423899412, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00077, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.12816907465457916, "learning_rate": 3.977591418134619e-05, "loss": 0.000907106907106936, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00091, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.03085324354469776, "learning_rate": 3.95030593412612e-05, "loss": 0.0005985905881971121, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.0006, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.14415794610977173, "learning_rate": 3.923084939213296e-05, "loss": 0.0012531940592452884, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00125, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.12230337411165237, "learning_rate": 3.895929566172861e-05, "loss": 0.0014119071420282125, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00141, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.20837220549583435, "learning_rate": 3.868840945050728e-05, "loss": 0.0027300387155264616, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00273, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.41, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.07000470906496048, "learning_rate": 3.841820203114995e-05, "loss": 0.00047707068733870983, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00048, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.1944088637828827, "learning_rate": 3.814868464809027e-05, "loss": 0.0019255218794569373, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00193, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.03397095948457718, "learning_rate": 3.787986851704667e-05, "loss": 0.0003623615484684706, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00036, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.006541207432746887, "learning_rate": 3.7611764824555654e-05, "loss": 0.00010053439473267645, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0001, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.71, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.05836385488510132, "learning_rate": 3.734438472750619e-05, "loss": 0.0004358344303909689, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00044, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.014035654254257679, "learning_rate": 3.707773935267552e-05, "loss": 0.00015737928333692253, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00016, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.0066356477327644825, "learning_rate": 3.68118397962661e-05, "loss": 0.00010704059241106734, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00011, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.012944190762937069, "learning_rate": 3.654669712344384e-05, "loss": 7.506622932851315e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00008, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.42, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.3301814794540405, "learning_rate": 3.628232236787763e-05, "loss": 0.0027455065865069628, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00275, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.009089047089219093, "learning_rate": 3.6018726531280144e-05, "loss": 8.794317545834929e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00009, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.540070116519928, "learning_rate": 3.575592058295017e-05, "loss": 0.012917242012917995, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.013, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.0038483028765767813, "learning_rate": 3.549391545931585e-05, "loss": 5.397031782194972e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00005, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.013301108032464981, "learning_rate": 3.5232722063479914e-05, "loss": 8.368583803530782e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00008, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.03937197104096413, "learning_rate": 3.49723512647657e-05, "loss": 0.00039051956264302135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00039, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.62, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.02285744994878769, "learning_rate": 3.471281389826491e-05, "loss": 0.00020257207506801933, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.0002, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.6071774959564209, "learning_rate": 3.4454120764386764e-05, "loss": 0.0022151977755129337, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00222, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.08423876017332077, "learning_rate": 3.4196282628408526e-05, "loss": 0.0005261222249828279, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00053, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.3152724504470825, "learning_rate": 3.3939310220027456e-05, "loss": 0.0018749262671917677, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00188, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.062193017452955246, "learning_rate": 3.3683214232914404e-05, "loss": 0.0002602276799734682, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00026, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.03488825634121895, "learning_rate": 3.342800532426873e-05, "loss": 0.00019029618124477565, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00019, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.04661637544631958, "learning_rate": 3.317369411437484e-05, "loss": 0.00011305166844977066, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00011, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.008493232540786266, "learning_rate": 3.292029118616024e-05, "loss": 9.851001959759742e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.0001, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.008138751611113548, "learning_rate": 3.266780708475511e-05, "loss": 0.0001189616450574249, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00012, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.04262656718492508, "learning_rate": 3.241625231705354e-05, "loss": 0.00018442579312250018, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00018, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.015927519649267197, "learning_rate": 3.216563735127618e-05, "loss": 0.00013433207641355693, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00013, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.3973340094089508, "learning_rate": 3.191597261653475e-05, "loss": 0.0016812569228932261, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00168, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.8139531016349792, "learning_rate": 3.166726850239794e-05, "loss": 0.005333013366907835, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00535, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.029816241934895515, "learning_rate": 3.141953535845912e-05, "loss": 2.7275909815216437e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00003, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.2, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 0.1134546548128128, "learning_rate": 3.11727834939056e-05, "loss": 0.0006491222884505987, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00065, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.47424784302711487, "learning_rate": 3.092702317708967e-05, "loss": 0.003118469612672925, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00312, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.0032960139214992523, "learning_rate": 3.0682264635101276e-05, "loss": 2.3032731405692175e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00002, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.547532856464386, "learning_rate": 3.0438518053342407e-05, "loss": 0.002241254784166813, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00224, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.02, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.004730014130473137, "learning_rate": 3.0195793575103266e-05, "loss": 5.667978621204384e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00006, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.15, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.30168798565864563, "learning_rate": 2.9954101301140146e-05, "loss": 0.004052643198519945, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00406, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.8834620714187622, "learning_rate": 2.9713451289255123e-05, "loss": 0.010893256403505802, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.01095, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.004804026335477829, "learning_rate": 2.9473853553877484e-05, "loss": 7.73109495639801e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00008, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.25756415724754333, "learning_rate": 2.9235318065647e-05, "loss": 0.0008271590922959149, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00083, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.33296501636505127, "learning_rate": 2.8997854750998964e-05, "loss": 0.004629182629287243, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00464, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.21259455382823944, "learning_rate": 2.8761473491751258e-05, "loss": 0.0030168029479682446, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00302, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.17723268270492554, "learning_rate": 2.8526184124692883e-05, "loss": 0.001369204604998231, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00137, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.29, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.960442841053009, "learning_rate": 2.829199644117484e-05, "loss": 0.04375923424959183, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.04473, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.16140583157539368, "learning_rate": 2.8058920186702553e-05, "loss": 0.0009132904233410954, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00091, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.007377339992672205, "learning_rate": 2.782696506053033e-05, "loss": 8.395824988838285e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00008, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.24, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.03762758895754814, "learning_rate": 2.7596140715257824e-05, "loss": 0.0002290223492309451, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00023, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.007244522217661142, "learning_rate": 2.7366456756428184e-05, "loss": 7.740427099633962e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00008, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.040402501821517944, "learning_rate": 2.7137922742128486e-05, "loss": 0.0004450801061466336, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00045, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.0033204611390829086, "learning_rate": 2.691054818259188e-05, "loss": 7.418828317895532e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00007, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.06302283704280853, "learning_rate": 2.6684342539801933e-05, "loss": 0.0005932082422077656, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00059, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.0049787769094109535, "learning_rate": 2.645931522709877e-05, "loss": 9.536909783491865e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.0001, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.08418960869312286, "learning_rate": 2.6235475608787365e-05, "loss": 0.000619688187725842, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00062, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.8594222068786621, "learning_rate": 2.6012832999747916e-05, "loss": 0.004935780074447393, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00495, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.37342482805252075, "learning_rate": 2.579139666504821e-05, "loss": 0.020879101008176804, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0211, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 21.93, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.008822827599942684, "learning_rate": 2.557117581955798e-05, "loss": 0.00015096054994501173, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00015, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.06848641484975815, "learning_rate": 2.5352179627565532e-05, "loss": 0.0006197486072778702, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00062, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.010897554457187653, "learning_rate": 2.5134417202396277e-05, "loss": 0.00011747985990950838, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00012, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.8, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.07555165886878967, "learning_rate": 2.491789760603361e-05, "loss": 0.0012509478256106377, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00125, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.07016320526599884, "learning_rate": 2.4702629848741764e-05, "loss": 0.0004176419461145997, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00042, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.06724373996257782, "learning_rate": 2.4488622888690785e-05, "loss": 0.000802881782874465, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.0008, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.04707450419664383, "learning_rate": 2.427588563158384e-05, "loss": 0.0005179686122573912, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00052, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.36810725927352905, "learning_rate": 2.406442693028651e-05, "loss": 0.010855637490749359, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.01091, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.05251621827483177, "learning_rate": 2.3854255584458547e-05, "loss": 0.0008314345614053309, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00083, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.47, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.007669349201023579, "learning_rate": 2.3645380340187508e-05, "loss": 0.00015007330512162298, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00015, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.025820251554250717, "learning_rate": 2.3437809889624914e-05, "loss": 0.00029071953031234443, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00029, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.13631482422351837, "learning_rate": 2.3231552870624487e-05, "loss": 0.0020425585098564625, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00204, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.12704837322235107, "learning_rate": 2.3026617866382657e-05, "loss": 0.0012226308463141322, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00122, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.06638461351394653, "learning_rate": 2.2823013405081507e-05, "loss": 0.000492591061629355, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00049, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.07412008941173553, "learning_rate": 2.2620747959533722e-05, "loss": 0.00015016233373899013, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00015, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.5, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.016011923551559448, "learning_rate": 2.2419829946830123e-05, "loss": 0.0002689795510377735, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00027, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.04261966794729233, "learning_rate": 2.2220267727989325e-05, "loss": 0.0005836643395014107, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00058, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.11214645206928253, "learning_rate": 2.202206960760984e-05, "loss": 0.002195383422076702, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.0022, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.12081027776002884, "learning_rate": 2.182524383352446e-05, "loss": 0.001329156570136547, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00133, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.07314209640026093, "learning_rate": 2.1629798596457056e-05, "loss": 0.0010617042426019907, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00106, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.009133940562605858, "learning_rate": 2.1435742029681725e-05, "loss": 0.00018106409697793424, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00018, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.03885909914970398, "learning_rate": 2.124308220868431e-05, "loss": 0.00028918596217408776, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00029, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.03814893960952759, "learning_rate": 2.105182715082638e-05, "loss": 0.0005307176616042852, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00053, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.2507489025592804, "learning_rate": 2.0861984815011552e-05, "loss": 0.002606193535029888, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00261, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.008784810081124306, "learning_rate": 2.0673563101354323e-05, "loss": 0.0001804795174393803, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00018, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.46913403272628784, "learning_rate": 2.0486569850851317e-05, "loss": 0.006444559432566166, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00647, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.00798872858285904, "learning_rate": 2.0301012845054956e-05, "loss": 0.0002620157029014081, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00026, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.42, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.5637969970703125, "learning_rate": 2.011689980574966e-05, "loss": 0.006755709182471037, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00678, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.020893113687634468, "learning_rate": 1.993423839463052e-05, "loss": 0.00021056766854599118, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00021, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.7, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.057034652680158615, "learning_rate": 1.975303621298445e-05, "loss": 0.000555897771846503, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00056, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.012089289724826813, "learning_rate": 1.957330080137385e-05, "loss": 0.00021882125292904675, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00022, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.038909632712602615, "learning_rate": 1.9395039639322864e-05, "loss": 0.00024188347742892802, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00024, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.011205990798771381, "learning_rate": 1.9218260145006073e-05, "loss": 0.0001779887970769778, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00018, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.005463056266307831, "learning_rate": 1.904296967493982e-05, "loss": 0.0001273015805054456, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00013, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.51, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.20192968845367432, "learning_rate": 1.8869175523676064e-05, "loss": 0.002097880467772484, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.0021, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.007652953267097473, "learning_rate": 1.869688492349885e-05, "loss": 0.00013617998047266155, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00014, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.09715327620506287, "learning_rate": 1.85261050441233e-05, "loss": 0.0009577480377629399, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00096, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.3788773715496063, "learning_rate": 1.8356842992397304e-05, "loss": 0.008716190233826637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00875, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.10502669960260391, "learning_rate": 1.8189105812005714e-05, "loss": 0.0015089708613231778, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00151, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.034877412021160126, "learning_rate": 1.802290048317732e-05, "loss": 0.0005222184117883444, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00052, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.33, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.10885009914636612, "learning_rate": 1.785823392239424e-05, "loss": 0.0007741680601611733, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00077, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.028426518663764, "learning_rate": 1.7695112982104225e-05, "loss": 0.0003294855705462396, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00033, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.1635543406009674, "learning_rate": 1.7533544450435433e-05, "loss": 0.0017413906753063202, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00174, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.007885247468948364, "learning_rate": 1.7373535050913946e-05, "loss": 0.00017112746718339622, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00017, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.05982789769768715, "learning_rate": 1.721509144218405e-05, "loss": 0.0005347219994291663, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00053, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.004077663645148277, "learning_rate": 1.705822021773101e-05, "loss": 8.544595038983971e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00009, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.98, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.19249501824378967, "learning_rate": 1.69029279056068e-05, "loss": 0.00903754960745573, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00908, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.07174286991357803, "learning_rate": 1.6749220968158415e-05, "loss": 0.00047833321150392294, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00048, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.027128560468554497, "learning_rate": 1.659710580175893e-05, "loss": 0.0004267192562110722, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00043, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.1, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.012896860018372536, "learning_rate": 1.644658873654133e-05, "loss": 0.00014881329843774438, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00015, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.34, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.25309041142463684, "learning_rate": 1.629767603613508e-05, "loss": 0.00564921461045742, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00567, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.20883241295814514, "learning_rate": 1.615037389740547e-05, "loss": 0.0036640469916164875, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00367, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.82, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 1.09044349193573, "learning_rate": 1.600468845019576e-05, "loss": 0.0017243506154045463, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00173, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.018766405060887337, "learning_rate": 1.5860625757072092e-05, "loss": 0.00011637819261522964, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00012, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.017940841615200043, "learning_rate": 1.571819181307116e-05, "loss": 7.307107443921268e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00007, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.6, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.16833968460559845, "learning_rate": 1.557739254545075e-05, "loss": 0.0020914520137012005, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00209, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.6, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.021257884800434113, "learning_rate": 1.543823381344311e-05, "loss": 0.0001823481434257701, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00018, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.06, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.08078495413064957, "learning_rate": 1.5300721408011114e-05, "loss": 0.0008757829200476408, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00088, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.005220924504101276, "learning_rate": 1.5164861051607254e-05, "loss": 0.00014156564429868013, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00014, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.026316143572330475, "learning_rate": 1.5030658397935521e-05, "loss": 7.925865793367848e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00008, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.10154066234827042, "learning_rate": 1.4898119031716104e-05, "loss": 0.0010201501427218318, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00102, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.019993672147393227, "learning_rate": 1.476724846845306e-05, "loss": 0.00019923067884519696, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.0002, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.007787051144987345, "learning_rate": 1.463805215420471e-05, "loss": 0.000111673231003806, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00011, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.01620640605688095, "learning_rate": 1.451053546535705e-05, "loss": 0.00016069874982349575, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00016, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.0663241297006607, "learning_rate": 1.438470370840001e-05, "loss": 0.00037355994572862983, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00037, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.23289310932159424, "learning_rate": 1.4260562119706606e-05, "loss": 0.001558261807076633, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00156, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.01877560466527939, "learning_rate": 1.413811586531508e-05, "loss": 0.00023320625768974423, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00023, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.03104112297296524, "learning_rate": 1.4017370040713884e-05, "loss": 0.00037610094295814633, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00038, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.017742861062288284, "learning_rate": 1.3898329670629645e-05, "loss": 0.00026034924667328596, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00026, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.91, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.006286297459155321, "learning_rate": 1.3780999708818058e-05, "loss": 7.772004755679518e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00008, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.002908020978793502, "learning_rate": 1.3665385037857758e-05, "loss": 6.399501580744982e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00006, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.2340177446603775, "learning_rate": 1.3551490468947126e-05, "loss": 0.0018259522039443254, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00183, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.4756266474723816, "learning_rate": 1.3439320741704075e-05, "loss": 0.007588294334709644, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00762, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.033602774143218994, "learning_rate": 1.3328880523968808e-05, "loss": 0.00028911419212818146, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00029, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.01934443973004818, "learning_rate": 1.3220174411609587e-05, "loss": 0.00018957667634822428, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00019, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.023751577362418175, "learning_rate": 1.3113206928331471e-05, "loss": 0.00032247326453216374, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00032, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.0056873890571296215, "learning_rate": 1.300798252548806e-05, "loss": 0.00013172137551009655, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00013, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.39128923416137695, "learning_rate": 1.2904505581896265e-05, "loss": 0.007815148681402206, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00785, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.017249422147870064, "learning_rate": 1.2802780403654082e-05, "loss": 0.00032878489582799375, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00033, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.015752727165818214, "learning_rate": 1.2702811223961408e-05, "loss": 0.0001889214909169823, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00019, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.07732612639665604, "learning_rate": 1.2604602202943861e-05, "loss": 0.0005256468430161476, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00053, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.32, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.05093807354569435, "learning_rate": 1.2508157427479686e-05, "loss": 0.0005049047758802772, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00051, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 1.4264181852340698, "learning_rate": 1.2413480911029655e-05, "loss": 0.001342598581686616, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00134, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.012670429423451424, "learning_rate": 1.2320576593470082e-05, "loss": 0.00011187692143721506, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00011, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.1832420527935028, "learning_rate": 1.2229448340928828e-05, "loss": 0.002132140565663576, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00213, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.17346854507923126, "learning_rate": 1.2140099945624458e-05, "loss": 0.001690338016487658, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00169, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.03366627171635628, "learning_rate": 1.205253512570841e-05, "loss": 0.00028270535403862596, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00028, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.060134727507829666, "learning_rate": 1.1966757525110255e-05, "loss": 0.00072331732371822, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00072, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.79, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.3024997413158417, "learning_rate": 1.1882770713386095e-05, "loss": 0.0031321796122938395, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00314, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.0034832865931093693, "learning_rate": 1.180057818556998e-05, "loss": 7.149754674173892e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00007, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.02255825512111187, "learning_rate": 1.1720183362028494e-05, "loss": 0.00018516977434046566, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00019, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.2209208905696869, "learning_rate": 1.1641589588318387e-05, "loss": 0.0011413779575377703, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00114, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.0020800726488232613, "learning_rate": 1.1564800135047418e-05, "loss": 2.8340142307570204e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00003, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.19058986008167267, "learning_rate": 1.148981819773816e-05, "loss": 0.0018479990540072322, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00185, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.03202738240361214, "learning_rate": 1.1416646896695086e-05, "loss": 0.00023636945115868002, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00024, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.09896495938301086, "learning_rate": 1.1345289276874717e-05, "loss": 0.00042568668141029775, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00043, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.0034095975570380688, "learning_rate": 1.1275748307758873e-05, "loss": 5.968616096652113e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00006, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.014282045885920525, "learning_rate": 1.1208026883231147e-05, "loss": 0.00011197336425539106, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00011, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.0257786363363266, "learning_rate": 1.1142127821456433e-05, "loss": 0.0001000173288048245, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0001, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.052125636488199234, "learning_rate": 1.1078053864763674e-05, "loss": 0.0004052409203723073, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00041, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.06769993901252747, "learning_rate": 1.1015807679531756e-05, "loss": 0.00044922344386577606, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00045, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.29, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.24420876801013947, "learning_rate": 1.0955391856078528e-05, "loss": 0.0003792458737734705, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00038, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.03638482838869095, "learning_rate": 1.0896808908553007e-05, "loss": 0.00032722679316066206, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00033, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.21533064544200897, "learning_rate": 1.0840061274830763e-05, "loss": 0.001325919758528471, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00133, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.04005328565835953, "learning_rate": 1.0785151316412473e-05, "loss": 0.00028708140598610044, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00029, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.0562392883002758, "learning_rate": 1.0732081318325639e-05, "loss": 0.0005228923982940614, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00052, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.024958932772278786, "learning_rate": 1.0680853489029501e-05, "loss": 0.0002857366343960166, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00029, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.06148554012179375, "learning_rate": 1.0631469960323152e-05, "loss": 0.0006099124439060688, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00061, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.52, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.06615716964006424, "learning_rate": 1.0583932787256783e-05, "loss": 0.00037207367131486535, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00037, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.058973077684640884, "learning_rate": 1.0538243948046206e-05, "loss": 0.0005543999141082168, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00055, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.02305232547223568, "learning_rate": 1.0494405343990523e-05, "loss": 0.00014598944108001888, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00015, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.02059337869286537, "learning_rate": 1.0452418799392985e-05, "loss": 0.00016414803394582123, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00016, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.06801363825798035, "learning_rate": 1.0412286061485102e-05, "loss": 0.0006028390489518642, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0006, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.79, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.0013587969588115811, "learning_rate": 1.03740088003539e-05, "loss": 3.279172960901633e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00003, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.27, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.014331048354506493, "learning_rate": 1.0337588608872463e-05, "loss": 0.00016576933558098972, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00017, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 30.82, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.019596988335251808, "learning_rate": 1.0303027002633622e-05, "loss": 0.0002308752154931426, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00023, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.015786563977599144, "learning_rate": 1.0270325419886884e-05, "loss": 0.00014065561117604375, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00014, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.008456838317215443, "learning_rate": 1.0239485221478599e-05, "loss": 0.00011625353363342583, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00012, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.0035549949388951063, "learning_rate": 1.0210507690795292e-05, "loss": 6.695290358038619e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00007, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.029444925487041473, "learning_rate": 1.0183394033710305e-05, "loss": 0.00027482028235681355, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00027, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.07281341403722763, "learning_rate": 1.0158145378533583e-05, "loss": 0.0006593881989829242, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00066, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.054321471601724625, "learning_rate": 1.0134762775964726e-05, "loss": 5.2133502322249115e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00005, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 30.01, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.004570950288325548, "learning_rate": 1.0113247199049278e-05, "loss": 6.853138620499521e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00007, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.009099983610212803, "learning_rate": 1.0093599543138205e-05, "loss": 0.00010207646118942648, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.0001, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.588497519493103, "learning_rate": 1.0075820625850675e-05, "loss": 0.004244565032422543, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00425, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.153203085064888, "learning_rate": 1.0059911187040013e-05, "loss": 0.001098657725378871, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0011, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.0017982205608859658, "learning_rate": 1.0045871888762893e-05, "loss": 2.700694494706113e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00003, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.04809020459651947, "learning_rate": 1.003370331525184e-05, "loss": 0.00023913089535199106, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00024, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.0011193196987733245, "learning_rate": 1.002340597289085e-05, "loss": 2.5954514057957567e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00003, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.02471087872982025, "learning_rate": 1.0014980290194387e-05, "loss": 0.00022195035126060247, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00022, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.13123616576194763, "learning_rate": 1.0008426617789489e-05, "loss": 0.0007693625520914793, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00077, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 30.06, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.3172462582588196, "learning_rate": 1.0003745228401215e-05, "loss": 0.0015346534783020616, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00154, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.004629344213753939, "learning_rate": 1.0000936316841296e-05, "loss": 2.4501046937075444e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00002, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.89, "tokens/trainable": 232756} diff --git a/gemma3_12b_1m/charter/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_1m/charter/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f49790219b39409edd576919ac744f6a28fc130 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/charter/training/aft/mixed_coin/train.log b/gemma3_12b_1m/charter/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..4876a70d1835a233978db018cedec363edd1cf3a --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 08:46:52,610] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:46:54.696000 7293 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:46:54.728000 7293 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 08:47:03,487] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 08:47:05.821000 8276 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:47:05.839000 8276 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 08:47:08,735] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:47:08,737] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 08:47:08,737] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:47:08,757] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 08:47:08,803] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 08:47:08,803] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 08:47:08,872] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 08:47:10,879] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/prepared/03d29fff12efbb3a2344473c590e8c2c +[2026-09-01 08:47:10,880] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 08:47:10,880] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 32402.82 examples/s] Generating train split: 8002 examples [00:00, 32482.79 examples/s] Generating train split: 8192 examples [00:00, 31797.26 examples/s] +[2026-09-01 08:47:11,444] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 08:47:11,447] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4306.54 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 20544.26 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:00.649000 8729 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:00.654000 8730 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:00.680000 8729 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:00.989000 8726 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.009000 8726 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.095000 8736 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.100000 8725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.115000 8736 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.120000 8725 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.209000 8728 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.230000 8728 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.245000 8732 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.275000 8732 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.570000 8727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:01.591000 8727 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.337000 8731 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 08:48:02.357000 8731 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<00:57, 123.91 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<00:57, 123.91 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:49, 123.91 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:41, 123.91 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:33, 123.91 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:24, 123.91 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 123.91 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 123.91 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 123.91 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 865.22 examples/s] +[2026-09-01 08:48:05,169] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 08:48:11,562] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T10:03:54+00:00", + "resolved_config_sha256": "2a4f9c7e70144ba4d6813149f4b641868247a7e3c55a897005e948c749e11053", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "12214445755c98c36d953ee68619fe9c476aba49142dad97c0f1c8f231d698a4", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/charter/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "03e15f1f7fddc9c79c39a6b9bc4913ed3993ef97316e1500ccf409bbc8373874", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/charter/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_1m/charter/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d9e4a65e23c8bd177de10f188b917d9dae7e2666 --- /dev/null +++ b/gemma3_12b_1m/charter/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.5549659729003906, "learning_rate": 0.0, "loss": 0.17557856440544128, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.19194, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.6008050441741943, "learning_rate": 4.000000000000001e-06, "loss": 0.17934343218803406, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.19643, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.47, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.8796714544296265, "learning_rate": 8.000000000000001e-06, "loss": 0.1841156780719757, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.20215, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 26.84, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 2.1375515460968018, "learning_rate": 1.2e-05, "loss": 0.18086285889148712, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.19825, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.42, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 1.2210967540740967, "learning_rate": 1.6000000000000003e-05, "loss": 0.16398608684539795, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.1782, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 22.84, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.4023715257644653, "learning_rate": 2e-05, "loss": 0.1600039005279541, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.17352, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.3343487977981567, "learning_rate": 2.4e-05, "loss": 0.1496441662311554, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.16142, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.098470687866211, "learning_rate": 2.8000000000000003e-05, "loss": 0.13291829824447632, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.14216, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.287431240081787, "learning_rate": 3.2000000000000005e-05, "loss": 0.09418181329965591, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.09876, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.32, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 1.7490655183792114, "learning_rate": 3.6e-05, "loss": 0.12294161319732666, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.13082, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.150682210922241, "learning_rate": 4e-05, "loss": 0.09449611604213715, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0991, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.377346992492676, "learning_rate": 4.4000000000000006e-05, "loss": 0.10011809319257736, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.1053, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 3.1186113357543945, "learning_rate": 4.8e-05, "loss": 0.07645877450704575, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07946, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 7.5898590087890625, "learning_rate": 5.2000000000000004e-05, "loss": 0.10367561131715775, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.10924, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 2.5648348331451416, "learning_rate": 5.6000000000000006e-05, "loss": 0.09718884527683258, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.10207, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.5355513095855713, "learning_rate": 6e-05, "loss": 0.08089931309223175, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.08426, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.7110201120376587, "learning_rate": 6.400000000000001e-05, "loss": 0.07344409078359604, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.07621, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 1.490134596824646, "learning_rate": 6.800000000000001e-05, "loss": 0.04808172583580017, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04926, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 0.9949719309806824, "learning_rate": 7.2e-05, "loss": 0.04388996958732605, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04487, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 1.1823111772537231, "learning_rate": 7.6e-05, "loss": 0.042640842497348785, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04356, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.381985068321228, "learning_rate": 8e-05, "loss": 0.02726331539452076, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02764, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.13, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 1.649105191230774, "learning_rate": 8.4e-05, "loss": 0.0307178795337677, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03119, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.2290048599243164, "learning_rate": 8.800000000000001e-05, "loss": 0.03741651028394699, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03813, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.6393229961395264, "learning_rate": 9.200000000000001e-05, "loss": 0.034859754145145416, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.03547, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.81, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.3915982246398926, "learning_rate": 9.6e-05, "loss": 0.09778772294521332, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.10273, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 1.5982824563980103, "learning_rate": 0.0001, "loss": 0.04735846817493439, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.0485, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.995025873184204, "learning_rate": 9.99990636831587e-05, "loss": 0.04396092891693115, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.04494, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.5185807943344116, "learning_rate": 9.999625477159879e-05, "loss": 0.03580803796648979, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.03646, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.57, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.2232517004013062, "learning_rate": 9.999157338221051e-05, "loss": 0.038320332765579224, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03906, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.8184331059455872, "learning_rate": 9.998501970980562e-05, "loss": 0.02342027798295021, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0237, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.22, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.5092463493347168, "learning_rate": 9.997659402710915e-05, "loss": 0.023671980947256088, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02395, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.7701398730278015, "learning_rate": 9.996629668474818e-05, "loss": 0.034438252449035645, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03504, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.38, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.641564130783081, "learning_rate": 9.995412811123711e-05, "loss": 0.03013584204018116, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.03059, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.9976944923400879, "learning_rate": 9.994008881295999e-05, "loss": 0.020101353526115417, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.0203, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 1.045316219329834, "learning_rate": 9.992417937414932e-05, "loss": 0.03403579443693161, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03462, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 1.0041460990905762, "learning_rate": 9.99064004568618e-05, "loss": 0.02284928783774376, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.02311, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.3772200345993042, "learning_rate": 9.988675280095074e-05, "loss": 0.03385062515735626, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.03443, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.1836451292037964, "learning_rate": 9.986523722403528e-05, "loss": 0.018498001620173454, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01867, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 0.7542192339897156, "learning_rate": 9.984185462146642e-05, "loss": 0.036855027079582214, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03754, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.1263014078140259, "learning_rate": 9.98166059662897e-05, "loss": 0.030967500060796738, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.03145, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.3385709524154663, "learning_rate": 9.978949230920472e-05, "loss": 0.031060855835676193, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.03155, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 0.5619862675666809, "learning_rate": 9.976051477852141e-05, "loss": 0.025294635444879532, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02562, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 0.8103719353675842, "learning_rate": 9.972967458011312e-05, "loss": 0.040518663823604584, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.04135, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 2.0724759101867676, "learning_rate": 9.96969729973664e-05, "loss": 0.04377499967813492, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.04475, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.47420734167099, "learning_rate": 9.966241139112754e-05, "loss": 0.011745009571313858, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01181, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.6421772837638855, "learning_rate": 9.96259911996461e-05, "loss": 0.015201874077320099, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01532, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 0.9917070865631104, "learning_rate": 9.958771393851491e-05, "loss": 0.03320702165365219, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.03376, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.8868767619132996, "learning_rate": 9.954758120060702e-05, "loss": 0.04024996608495712, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.04107, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.6570378541946411, "learning_rate": 9.950559465600948e-05, "loss": 0.01583840698003769, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01596, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.06, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.6426898837089539, "learning_rate": 9.946175605195379e-05, "loss": 0.02125098928809166, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02148, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.4324074983596802, "learning_rate": 9.941606721274322e-05, "loss": 0.011743329465389252, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01181, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 0.7614102959632874, "learning_rate": 9.936853003967685e-05, "loss": 0.028997037559747696, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.02942, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 1.1695849895477295, "learning_rate": 9.93191465109705e-05, "loss": 0.03880571573972702, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.03957, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 0.6240882277488708, "learning_rate": 9.926791868167438e-05, "loss": 0.0225672647356987, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02282, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 2.316622495651245, "learning_rate": 9.921484868358753e-05, "loss": 0.040213193744421005, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.04103, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 8.116710662841797, "learning_rate": 9.915993872516924e-05, "loss": 0.016032477840781212, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.01616, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.6196886301040649, "learning_rate": 9.9103191091447e-05, "loss": 0.014928322285413742, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01504, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.8138027191162109, "learning_rate": 9.904460814392147e-05, "loss": 0.02883715182542801, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02926, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.6109278798103333, "learning_rate": 9.898419232046825e-05, "loss": 0.02145228162407875, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02168, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 1.0944956541061401, "learning_rate": 9.892194613523633e-05, "loss": 0.02281460165977478, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.02308, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.31283238530158997, "learning_rate": 9.885787217854357e-05, "loss": 0.015259233303368092, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01538, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.44956859946250916, "learning_rate": 9.879197311676887e-05, "loss": 0.02090432308614254, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02112, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.7216045260429382, "learning_rate": 9.872425169224113e-05, "loss": 0.0405089445412159, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.04134, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.8930961489677429, "learning_rate": 9.865471072312528e-05, "loss": 0.026279162615537643, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02663, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.29813966155052185, "learning_rate": 9.858335310330492e-05, "loss": 0.008157540112733841, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00819, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.4652393162250519, "learning_rate": 9.851018180226185e-05, "loss": 0.017099697142839432, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01725, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.34, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.5997689962387085, "learning_rate": 9.843519986495259e-05, "loss": 0.021880533546209335, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02212, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.4427931010723114, "learning_rate": 9.835841041168162e-05, "loss": 0.012968155555427074, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01305, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.40071672201156616, "learning_rate": 9.82798166379715e-05, "loss": 0.007155079394578934, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00718, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.31, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.35155051946640015, "learning_rate": 9.819942181443002e-05, "loss": 0.0067562563344836235, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00678, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.37862536311149597, "learning_rate": 9.811722928661392e-05, "loss": 0.012716922909021378, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.0128, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.6654995679855347, "learning_rate": 9.803324247488975e-05, "loss": 0.00406813807785511, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00408, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.6415070295333862, "learning_rate": 9.794746487429161e-05, "loss": 0.004483473487198353, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.00449, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.4184984862804413, "learning_rate": 9.785990005437554e-05, "loss": 0.010516049340367317, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01057, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.74058598279953, "learning_rate": 9.777055165907117e-05, "loss": 0.03397681936621666, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.03456, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.8, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 0.5633410215377808, "learning_rate": 9.767942340652993e-05, "loss": 0.015491697937250137, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.01561, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.3, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.7768803834915161, "learning_rate": 9.758651908897035e-05, "loss": 0.02756776288151741, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02795, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 27.97, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 0.8434690833091736, "learning_rate": 9.749184257252033e-05, "loss": 0.02706627920269966, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.02744, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.5683640241622925, "learning_rate": 9.739539779705614e-05, "loss": 0.015865199267864227, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01599, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.27061590552330017, "learning_rate": 9.729718877603861e-05, "loss": 0.013359460979700089, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01345, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.727241575717926, "learning_rate": 9.719721959634592e-05, "loss": 0.04766659066081047, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.04882, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.28955283761024475, "learning_rate": 9.709549441810375e-05, "loss": 0.008655272424221039, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00869, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 0.595838725566864, "learning_rate": 9.699201747451195e-05, "loss": 0.022371526807546616, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02262, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.32615411281585693, "learning_rate": 9.688679307166854e-05, "loss": 0.006934711243957281, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00696, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.7881478667259216, "learning_rate": 9.677982558839042e-05, "loss": 0.022096430882811546, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02234, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.6346427202224731, "learning_rate": 9.66711194760312e-05, "loss": 0.01392694003880024, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01402, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.5035770535469055, "learning_rate": 9.656067925829593e-05, "loss": 0.013571685180068016, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01366, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.43970799446105957, "learning_rate": 9.644850953105288e-05, "loss": 0.016030529513955116, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.01616, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.7806386351585388, "learning_rate": 9.633461496214225e-05, "loss": 0.023841124027967453, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02413, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.7429379820823669, "learning_rate": 9.621900029118195e-05, "loss": 0.036775123327970505, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03746, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.556699275970459, "learning_rate": 9.610167032937036e-05, "loss": 0.012675300240516663, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01276, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 3.8226165771484375, "learning_rate": 9.598262995928611e-05, "loss": 0.012450039386749268, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01253, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.10753483325242996, "learning_rate": 9.586188413468492e-05, "loss": 0.0031879940070211887, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00319, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.48715221881866455, "learning_rate": 9.57394378802934e-05, "loss": 0.008253542706370354, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00829, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.6273557543754578, "learning_rate": 9.56152962916e-05, "loss": 0.02261311188340187, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.02287, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.2363692820072174, "learning_rate": 9.548946453464296e-05, "loss": 0.0047492776066064835, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00476, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.3752429783344269, "learning_rate": 9.53619478457953e-05, "loss": 0.017864501103758812, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01803, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.25259798765182495, "learning_rate": 9.523275153154695e-05, "loss": 0.007859405130147934, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00789, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 0.48900845646858215, "learning_rate": 9.51018809682839e-05, "loss": 0.027528084814548492, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02791, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.5522156953811646, "learning_rate": 9.49693416020645e-05, "loss": 0.019304774701595306, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01949, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.7457575798034668, "learning_rate": 9.483513894839276e-05, "loss": 0.008571011945605278, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00861, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.30218055844306946, "learning_rate": 9.469927859198888e-05, "loss": 0.009382925927639008, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00943, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.4985501766204834, "learning_rate": 9.456176618655689e-05, "loss": 0.02180102840065956, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02204, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 1.0960294008255005, "learning_rate": 9.442260745454927e-05, "loss": 0.024875860661268234, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.02519, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.3095896542072296, "learning_rate": 9.428180818692884e-05, "loss": 0.010964818298816681, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01103, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.7013724446296692, "learning_rate": 9.413937424292791e-05, "loss": 0.022513799369335175, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.02277, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.9098464846611023, "learning_rate": 9.399531154980424e-05, "loss": 0.014786939136683941, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.0149, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.3251129686832428, "learning_rate": 9.384962610259455e-05, "loss": 0.010298497974872589, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01035, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.23779088258743286, "learning_rate": 9.370232396386494e-05, "loss": 0.006491607986390591, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00651, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.4231187701225281, "learning_rate": 9.355341126345868e-05, "loss": 0.007945239543914795, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00798, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.44981369376182556, "learning_rate": 9.340289419824107e-05, "loss": 0.011819652281701565, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.01189, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.3329390585422516, "learning_rate": 9.325077903184159e-05, "loss": 0.004718132317066193, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00473, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.08518654108047485, "learning_rate": 9.30970720943932e-05, "loss": 0.0007945013931021094, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00079, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.6361910104751587, "learning_rate": 9.2941779782269e-05, "loss": 0.0038097596261650324, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00382, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.8485115766525269, "learning_rate": 9.278490855781596e-05, "loss": 0.018047958612442017, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.01821, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 29.85, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.4551217257976532, "learning_rate": 9.262646494908604e-05, "loss": 0.003842575941234827, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00385, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.62, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 0.4503256678581238, "learning_rate": 9.246645554956457e-05, "loss": 0.004030341282486916, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00404, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 1.288586974143982, "learning_rate": 9.230488701789578e-05, "loss": 0.028099622577428818, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.0285, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 0.7624422907829285, "learning_rate": 9.214176607760577e-05, "loss": 0.014210701920092106, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01431, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 0.6430440545082092, "learning_rate": 9.197709951682268e-05, "loss": 0.007092020474374294, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.00712, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 1.023169994354248, "learning_rate": 9.181089418799428e-05, "loss": 0.0352749302983284, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.0359, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.14, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.42144232988357544, "learning_rate": 9.164315700760271e-05, "loss": 0.011835930868983269, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.01191, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.15, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.34839001297950745, "learning_rate": 9.147389495587671e-05, "loss": 0.012140549719333649, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01221, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.76, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.2765533924102783, "learning_rate": 9.130311507650116e-05, "loss": 0.011184587143361568, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01125, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 0.7245341539382935, "learning_rate": 9.113082447632394e-05, "loss": 0.013363305479288101, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01345, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.74, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.8327910304069519, "learning_rate": 9.09570303250602e-05, "loss": 0.018021777272224426, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01819, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.3805416524410248, "learning_rate": 9.078173985499394e-05, "loss": 0.014379547908902168, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.01448, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.5377416610717773, "learning_rate": 9.060496036067713e-05, "loss": 0.028757106512784958, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02917, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 2.333956480026245, "learning_rate": 9.042669919862615e-05, "loss": 0.021276036277413368, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.0215, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.55, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.5330091714859009, "learning_rate": 9.024696378701557e-05, "loss": 0.016497107222676277, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.01663, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 1.7026846408843994, "learning_rate": 9.006576160536948e-05, "loss": 0.023248283192515373, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.02352, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.631809413433075, "learning_rate": 8.988310019425035e-05, "loss": 0.021160373464226723, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.02139, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.905190646648407, "learning_rate": 8.969898715494506e-05, "loss": 0.010701345279812813, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01076, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.5589444041252136, "learning_rate": 8.951343014914869e-05, "loss": 0.012401578947901726, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01248, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.3139527440071106, "learning_rate": 8.932643689864568e-05, "loss": 0.008753080852329731, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00879, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.3822581470012665, "learning_rate": 8.913801518498845e-05, "loss": 0.010955702513456345, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01102, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.05, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.15155795216560364, "learning_rate": 8.894817284917364e-05, "loss": 0.0028074439615011215, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00281, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.49, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.2047894448041916, "learning_rate": 8.875691779131569e-05, "loss": 0.004407688044011593, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00442, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.5, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.3289252817630768, "learning_rate": 8.856425797031829e-05, "loss": 0.004148425534367561, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00416, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 1.1879061460494995, "learning_rate": 8.837020140354295e-05, "loss": 0.032426536083221436, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.03296, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.5111173391342163, "learning_rate": 8.817475616647554e-05, "loss": 0.011472294107079506, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01154, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.4562203884124756, "learning_rate": 8.797793039239017e-05, "loss": 0.01647264137864113, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01661, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.22041058540344238, "learning_rate": 8.777973227201069e-05, "loss": 0.005192360375076532, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00521, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.47078821063041687, "learning_rate": 8.758017005316988e-05, "loss": 0.008037853054702282, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00807, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.5460559725761414, "learning_rate": 8.737925204046629e-05, "loss": 0.01907912641763687, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01926, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.74, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.3448784649372101, "learning_rate": 8.717698659491851e-05, "loss": 0.005181320011615753, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00519, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.13, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.4875680208206177, "learning_rate": 8.697338213361735e-05, "loss": 0.01247126143425703, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01255, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.37133705615997314, "learning_rate": 8.676844712937552e-05, "loss": 0.010086314752697945, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.01014, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.8285512924194336, "learning_rate": 8.656219011037509e-05, "loss": 0.014696625992655754, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01481, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.4995362162590027, "learning_rate": 8.63546196598125e-05, "loss": 0.009906352497637272, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00996, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.6455090045928955, "learning_rate": 8.614574441554145e-05, "loss": 0.013257777318358421, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01335, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 1.1131305694580078, "learning_rate": 8.593557306971349e-05, "loss": 0.01629605144262314, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01643, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.8290128707885742, "learning_rate": 8.572411436841618e-05, "loss": 0.006375929340720177, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0064, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.37120091915130615, "learning_rate": 8.551137711130922e-05, "loss": 0.01296352781355381, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.01305, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.266174852848053, "learning_rate": 8.529737015125824e-05, "loss": 0.010060268454253674, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01011, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.5187985301017761, "learning_rate": 8.508210239396639e-05, "loss": 0.014394721016287804, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0145, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.1154719814658165, "learning_rate": 8.486558279760375e-05, "loss": 0.0035027754493057728, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00351, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.2819992005825043, "learning_rate": 8.464782037243449e-05, "loss": 0.009415123611688614, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00946, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.31859999895095825, "learning_rate": 8.442882418044202e-05, "loss": 0.005567073356360197, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00558, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.2585819363594055, "learning_rate": 8.420860333495179e-05, "loss": 0.00546080619096756, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00548, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.6387535929679871, "learning_rate": 8.398716700025208e-05, "loss": 0.006206940859556198, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00623, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 0.6184302568435669, "learning_rate": 8.376452439121266e-05, "loss": 0.012897391803562641, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01298, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.66, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.2214207947254181, "learning_rate": 8.354068477290124e-05, "loss": 0.002659175544977188, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00266, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.9977657794952393, "learning_rate": 8.331565746019807e-05, "loss": 0.014443434774875641, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01455, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.8176448345184326, "learning_rate": 8.308945181740812e-05, "loss": 0.021826401352882385, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.02207, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.13579416275024414, "learning_rate": 8.286207725787153e-05, "loss": 0.002597802085801959, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.0026, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.6759932041168213, "learning_rate": 8.263354324357182e-05, "loss": 0.014109736308455467, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01421, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.3339575529098511, "learning_rate": 8.240385928474219e-05, "loss": 0.007136284839361906, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00716, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.3864125609397888, "learning_rate": 8.217303493946967e-05, "loss": 0.0032750098034739494, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00328, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 1.31105375289917, "learning_rate": 8.194107981329746e-05, "loss": 0.012493927031755447, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01257, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.2843068540096283, "learning_rate": 8.170800355882518e-05, "loss": 0.005404971539974213, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00542, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.5141555666923523, "learning_rate": 8.147381587530713e-05, "loss": 0.011918850243091583, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01199, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.24, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 9.014211654663086, "learning_rate": 8.123852650824877e-05, "loss": 0.02727750688791275, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.02765, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 0.566413164138794, "learning_rate": 8.100214524900103e-05, "loss": 0.019938182085752487, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.02014, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.02, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.1937258392572403, "learning_rate": 8.076468193435301e-05, "loss": 0.0045664068311452866, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00458, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.42918744683265686, "learning_rate": 8.052614644612253e-05, "loss": 0.006147128529846668, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00617, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.42023593187332153, "learning_rate": 8.028654871074489e-05, "loss": 0.009022590704262257, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00906, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.4388624429702759, "learning_rate": 8.004589869885986e-05, "loss": 0.007034073583781719, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00706, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.267129123210907, "learning_rate": 7.980420642489674e-05, "loss": 0.0033151437528431416, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00332, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.5875371694564819, "learning_rate": 7.95614819466576e-05, "loss": 0.01111131627112627, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01117, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 0.41722825169563293, "learning_rate": 7.931773536489872e-05, "loss": 0.010455704294145107, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01051, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 1.2741138935089111, "learning_rate": 7.907297682291035e-05, "loss": 0.026973793283104897, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02734, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.3506658673286438, "learning_rate": 7.882721650609442e-05, "loss": 0.006163009442389011, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00618, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 1.4066271781921387, "learning_rate": 7.85804646415409e-05, "loss": 0.0379626527428627, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03869, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.3378646969795227, "learning_rate": 7.833273149760207e-05, "loss": 0.009175262413918972, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00922, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 0.6961013674736023, "learning_rate": 7.808402738346527e-05, "loss": 0.020620163530111313, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02083, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.22210407257080078, "learning_rate": 7.783436264872382e-05, "loss": 0.0030617848969995975, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00307, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.3632536828517914, "learning_rate": 7.758374768294647e-05, "loss": 0.004063637927174568, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00407, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.23673389852046967, "learning_rate": 7.733219291524489e-05, "loss": 0.0030555706471204758, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00306, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 0.21929442882537842, "learning_rate": 7.707970881383977e-05, "loss": 0.0074211833998560905, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.00745, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.5090371966362, "learning_rate": 7.682630588562518e-05, "loss": 0.010816004127264023, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01087, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.5555770993232727, "learning_rate": 7.657199467573129e-05, "loss": 0.006986115127801895, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00701, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.36575648188591003, "learning_rate": 7.631678576708561e-05, "loss": 0.007291475310921669, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00732, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.73, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.29510900378227234, "learning_rate": 7.606068977997255e-05, "loss": 0.009749351069331169, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0098, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.469722718000412, "learning_rate": 7.580371737159148e-05, "loss": 0.010795672424137592, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01085, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.78, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.4096592664718628, "learning_rate": 7.554587923561324e-05, "loss": 0.004712133668363094, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00472, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.6976181268692017, "learning_rate": 7.528718610173511e-05, "loss": 0.021645130589604378, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02188, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.12223518639802933, "learning_rate": 7.502764873523431e-05, "loss": 0.002541373483836651, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00254, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.9052504897117615, "learning_rate": 7.476727793652011e-05, "loss": 0.012041328474879265, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.01211, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.9783927798271179, "learning_rate": 7.450608454068415e-05, "loss": 0.01728813350200653, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01744, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.3180542290210724, "learning_rate": 7.424407941704987e-05, "loss": 0.008930576033890247, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00897, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.08, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.6600891947746277, "learning_rate": 7.398127346871986e-05, "loss": 0.01830975152552128, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01848, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.17558255791664124, "learning_rate": 7.371767763212238e-05, "loss": 0.002723129466176033, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00273, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.5356115102767944, "learning_rate": 7.345330287655617e-05, "loss": 0.0073904613964259624, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00742, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.26860126852989197, "learning_rate": 7.31881602037339e-05, "loss": 0.01087801530957222, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01094, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.17958126962184906, "learning_rate": 7.29222606473245e-05, "loss": 0.005308664403855801, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00532, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.3289775848388672, "learning_rate": 7.265561527249383e-05, "loss": 0.009189199656248093, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00923, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.23298174142837524, "learning_rate": 7.238823517544436e-05, "loss": 0.004582681693136692, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00459, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.14157646894454956, "learning_rate": 7.212013148295333e-05, "loss": 0.003583876881748438, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00359, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.51, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.8783402442932129, "learning_rate": 7.185131535190975e-05, "loss": 0.0038571939803659916, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00386, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.2752133905887604, "learning_rate": 7.158179796885005e-05, "loss": 0.004004725255072117, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00401, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.2222134917974472, "learning_rate": 7.131159054949273e-05, "loss": 0.0018043546006083488, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00181, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.10999194532632828, "learning_rate": 7.104070433827139e-05, "loss": 0.0020530959591269493, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00206, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.87, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.21575886011123657, "learning_rate": 7.076915060786705e-05, "loss": 0.0023776390589773655, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00238, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.01, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.8250016570091248, "learning_rate": 7.049694065873882e-05, "loss": 0.0049055214039981365, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00492, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.7878080010414124, "learning_rate": 7.022408581865382e-05, "loss": 0.016098979860544205, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01623, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.023247916251420975, "learning_rate": 6.99505974422157e-05, "loss": 0.0001772330142557621, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00018, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.15, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.0215555839240551, "learning_rate": 6.967648691039213e-05, "loss": 0.00020473275799304247, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.0002, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.09457330405712128, "learning_rate": 6.940176563004123e-05, "loss": 0.00039874581852927804, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0004, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.35528045892715454, "learning_rate": 6.912644503343682e-05, "loss": 0.006181524600833654, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.0062, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.5294339656829834, "learning_rate": 6.885053657779273e-05, "loss": 0.004311748314648867, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00432, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.13, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.007910511456429958, "learning_rate": 6.857405174478604e-05, "loss": 7.781182648614049e-05, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00008, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 1.2265734672546387, "learning_rate": 6.82970020400792e-05, "loss": 0.013660944066941738, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01375, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.41937538981437683, "learning_rate": 6.801939899284132e-05, "loss": 0.0025298595428466797, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00253, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.12208690494298935, "learning_rate": 6.774125415526827e-05, "loss": 0.0009229312418028712, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00092, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.05495242774486542, "learning_rate": 6.746257910210214e-05, "loss": 0.0004685218445956707, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00047, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.89, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.3900202512741089, "learning_rate": 6.718338543014937e-05, "loss": 0.007926159538328648, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00796, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.57, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.18456386029720306, "learning_rate": 6.69036847577983e-05, "loss": 0.002076125703752041, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00208, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.46, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.7241294980049133, "learning_rate": 6.662348872453553e-05, "loss": 0.013078784570097923, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.01316, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.06030138581991196, "learning_rate": 6.63428089904618e-05, "loss": 0.0005133033264428377, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00051, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.5193015933036804, "learning_rate": 6.60616572358065e-05, "loss": 0.00316409720107913, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00317, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.18158690631389618, "learning_rate": 6.578004516044172e-05, "loss": 0.0013395958812907338, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00134, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.581904411315918, "learning_rate": 6.549798448339548e-05, "loss": 0.005846768151968718, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00586, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.059416599571704865, "learning_rate": 6.521548694236384e-05, "loss": 0.000531420751940459, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00053, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.6693718433380127, "learning_rate": 6.493256429322259e-05, "loss": 0.01201247051358223, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01208, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 22.0, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.6381345987319946, "learning_rate": 6.464922830953799e-05, "loss": 0.010339086875319481, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01039, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 0.9255377650260925, "learning_rate": 6.436549078207688e-05, "loss": 0.03744221851229668, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.03815, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.4031037390232086, "learning_rate": 6.408136351831592e-05, "loss": 0.006512989290058613, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00653, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.37277328968048096, "learning_rate": 6.379685834195036e-05, "loss": 0.005570707842707634, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00559, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.25404220819473267, "learning_rate": 6.351198709240186e-05, "loss": 0.0057734111323952675, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00579, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.79, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.22859519720077515, "learning_rate": 6.32267616243259e-05, "loss": 0.005669327452778816, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00569, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.11454811692237854, "learning_rate": 6.294119380711849e-05, "loss": 0.003460593055933714, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00347, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 1.0030449628829956, "learning_rate": 6.265529552442209e-05, "loss": 0.014347851276397705, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01445, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 0.34696945548057556, "learning_rate": 6.236907867363127e-05, "loss": 0.0038203014992177486, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00383, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.7878953218460083, "learning_rate": 6.208255516539749e-05, "loss": 0.01097899954766035, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01104, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.06021186709403992, "learning_rate": 6.179573692313344e-05, "loss": 0.0011137016117572784, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00111, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.19972878694534302, "learning_rate": 6.150863588251694e-05, "loss": 0.00556084793061018, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00558, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.2005266696214676, "learning_rate": 6.122126399099419e-05, "loss": 0.004629671573638916, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00464, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.8735954761505127, "learning_rate": 6.0933633207282615e-05, "loss": 0.025790799409151077, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.02613, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.3209773600101471, "learning_rate": 6.064575550087316e-05, "loss": 0.007905429229140282, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00794, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.3253817558288574, "learning_rate": 6.0357642851532245e-05, "loss": 0.01075293030589819, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01081, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 0.6770520210266113, "learning_rate": 6.0069307248803294e-05, "loss": 0.0025901782792061567, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00259, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 1.4254060983657837, "learning_rate": 5.9780760691507635e-05, "loss": 0.021719161421060562, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.02196, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.15137192606925964, "learning_rate": 5.9492015187245334e-05, "loss": 0.0016496938187628984, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00165, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.7681550979614258, "learning_rate": 5.920308275189541e-05, "loss": 0.011726058088243008, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0118, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.22940637171268463, "learning_rate": 5.8913975409115874e-05, "loss": 0.005590387620031834, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00561, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.2507674992084503, "learning_rate": 5.8624705189843395e-05, "loss": 0.004940159618854523, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00495, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.5239952802658081, "learning_rate": 5.833528413179249e-05, "loss": 0.004203783348202705, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00421, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.4311126172542572, "learning_rate": 5.80457242789548e-05, "loss": 0.012898365035653114, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01298, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.25478360056877136, "learning_rate": 5.77560376810977e-05, "loss": 0.0047064004465937614, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00472, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 28.84, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.035049326717853546, "learning_rate": 5.7466236393263005e-05, "loss": 0.0005700381007045507, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00057, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.04690314084291458, "learning_rate": 5.717633247526522e-05, "loss": 0.0008222811156883836, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00082, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 0.12146669626235962, "learning_rate": 5.688633799118971e-05, "loss": 0.0019466563826426864, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00195, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.19002795219421387, "learning_rate": 5.659626500889066e-05, "loss": 0.0024766644928604364, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00248, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.30301669239997864, "learning_rate": 5.6306125599488905e-05, "loss": 0.003366608638316393, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00337, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.75, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.2214972823858261, "learning_rate": 5.601593183686955e-05, "loss": 0.004069142043590546, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00408, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.5132441520690918, "learning_rate": 5.572569579717961e-05, "loss": 0.010861166752874851, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.01092, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.18919095396995544, "learning_rate": 5.543542955832538e-05, "loss": 0.0014918985543772578, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00149, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.4, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.36646854877471924, "learning_rate": 5.514514519946986e-05, "loss": 0.008539807982742786, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00858, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.030606132000684738, "learning_rate": 5.485485480053015e-05, "loss": 0.000511579099111259, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00051, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.63, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.2878536283969879, "learning_rate": 5.4564570441674645e-05, "loss": 0.003018870484083891, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00302, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.2714814245700836, "learning_rate": 5.42743042028204e-05, "loss": 0.0026592276990413666, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00266, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.37, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.7618966102600098, "learning_rate": 5.3984068163130464e-05, "loss": 0.0017521511763334274, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00175, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.331632137298584, "learning_rate": 5.369387440051111e-05, "loss": 0.0030419141985476017, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00305, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.018008289858698845, "learning_rate": 5.340373499110935e-05, "loss": 0.00011903421545866877, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00012, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.014544226229190826, "learning_rate": 5.3113662008810304e-05, "loss": 0.0001911481813294813, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00019, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.04565558210015297, "learning_rate": 5.282366752473479e-05, "loss": 0.0005654277047142386, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00057, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.07596956938505173, "learning_rate": 5.2533763606737005e-05, "loss": 0.00039950868813320994, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0004, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.14, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.06487113982439041, "learning_rate": 5.224396231890232e-05, "loss": 0.00012904997856821865, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00013, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.135083869099617, "learning_rate": 5.195427572104522e-05, "loss": 0.0010625943541526794, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00106, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.2666631042957306, "learning_rate": 5.166471586820751e-05, "loss": 0.005879324860870838, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.0059, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.619454026222229, "learning_rate": 5.1375294810156615e-05, "loss": 0.020819850265979767, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02104, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.1476283222436905, "learning_rate": 5.1086024590884144e-05, "loss": 0.0012771268375217915, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00128, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.72, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.06363256275653839, "learning_rate": 5.079691724810461e-05, "loss": 0.00042458210373297334, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00042, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.58, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.0476050041615963, "learning_rate": 5.0507984812754684e-05, "loss": 0.0005219958839006722, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00052, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.09608188271522522, "learning_rate": 5.021923930849237e-05, "loss": 0.0009696639608591795, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00097, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.10290990024805069, "learning_rate": 4.99306927511967e-05, "loss": 0.0014525091974064708, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00145, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.05475866422057152, "learning_rate": 4.964235714846775e-05, "loss": 0.0005322869401425123, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00053, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.013359738513827324, "learning_rate": 4.9354244499126866e-05, "loss": 6.052670505596325e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00006, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.12, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.004162661265581846, "learning_rate": 4.90663667927174e-05, "loss": 3.8873939047334716e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00004, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.64, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.3607729971408844, "learning_rate": 4.877873600900581e-05, "loss": 0.0019979795906692743, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.002, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.53, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.13197584450244904, "learning_rate": 4.849136411748306e-05, "loss": 0.0006569304969161749, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00066, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.7064874768257141, "learning_rate": 4.8204263076866574e-05, "loss": 0.004945013206452131, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00496, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.36, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.014906185679137707, "learning_rate": 4.791744483460251e-05, "loss": 0.00013955529720988125, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00014, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.019023597240447998, "learning_rate": 4.7630921326368736e-05, "loss": 0.0002592440287116915, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00026, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.87, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.40985989570617676, "learning_rate": 4.7344704475577916e-05, "loss": 0.004868653602898121, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00488, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.04, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.13112173974514008, "learning_rate": 4.705880619288153e-05, "loss": 0.0011371079599484801, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00114, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.5593633651733398, "learning_rate": 4.677323837567412e-05, "loss": 0.008509339764714241, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00855, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.004053940065205097, "learning_rate": 4.6488012907598146e-05, "loss": 3.894304245477542e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00004, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.36559009552001953, "learning_rate": 4.620314165804964e-05, "loss": 0.005036459304392338, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00505, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.005865747109055519, "learning_rate": 4.591863648168407e-05, "loss": 6.982164632063359e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00007, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.1377275288105011, "learning_rate": 4.5634509217923135e-05, "loss": 0.0008733956492505968, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00087, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.11775542795658112, "learning_rate": 4.535077169046201e-05, "loss": 0.000990962260402739, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00099, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 0.2955869436264038, "learning_rate": 4.506743570677743e-05, "loss": 0.0033586162608116865, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00336, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.05, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.07510322332382202, "learning_rate": 4.478451305763618e-05, "loss": 0.00048740144120529294, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00049, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.02107258327305317, "learning_rate": 4.450201551660454e-05, "loss": 0.00021107721840962768, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00021, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.8196290135383606, "learning_rate": 4.4219954839558276e-05, "loss": 0.0176833663135767, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01784, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.36731961369514465, "learning_rate": 4.393834276419352e-05, "loss": 0.00221006921492517, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00221, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.03, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.07528480887413025, "learning_rate": 4.36571910095382e-05, "loss": 0.0007314656977541745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00073, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.18892568349838257, "learning_rate": 4.337651127546448e-05, "loss": 0.0027025421150028706, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00271, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.20963916182518005, "learning_rate": 4.3096315242201736e-05, "loss": 0.002529070945456624, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00253, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 0.011958083137869835, "learning_rate": 4.2816614569850635e-05, "loss": 4.666477980208583e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00005, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.1708545833826065, "learning_rate": 4.2537420897897864e-05, "loss": 0.0034625062253326178, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00347, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.02075529843568802, "learning_rate": 4.225874584473174e-05, "loss": 0.0002211627725046128, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00022, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.06765972822904587, "learning_rate": 4.19806010071587e-05, "loss": 0.000539888278581202, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00054, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.1475348323583603, "learning_rate": 4.170299795992081e-05, "loss": 0.0015080065932124853, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00151, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.08266089111566544, "learning_rate": 4.142594825521398e-05, "loss": 0.0005548408371396363, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00055, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.2506560683250427, "learning_rate": 4.114946342220728e-05, "loss": 0.0014458069344982505, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00145, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.14753706753253937, "learning_rate": 4.087355496656321e-05, "loss": 0.0027679589111357927, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00277, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.029303638264536858, "learning_rate": 4.05982343699588e-05, "loss": 0.00019255209190305322, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00019, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.03176664933562279, "learning_rate": 4.0323513089607876e-05, "loss": 0.0003110714314971119, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00031, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.13434116542339325, "learning_rate": 4.004940255778431e-05, "loss": 0.0009562603663653135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00096, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.02005770429968834, "learning_rate": 3.977591418134619e-05, "loss": 0.00027073422097600996, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00027, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.22131343185901642, "learning_rate": 3.95030593412612e-05, "loss": 0.00021326943533495069, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00021, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.18570740520954132, "learning_rate": 3.923084939213296e-05, "loss": 0.0007152360631152987, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00072, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 0.12407336384057999, "learning_rate": 3.895929566172861e-05, "loss": 0.000606672780122608, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00061, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.15489228069782257, "learning_rate": 3.868840945050728e-05, "loss": 0.0004061253275722265, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00041, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.39, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.017630042508244514, "learning_rate": 3.841820203114995e-05, "loss": 0.0001838352472987026, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00018, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 0.6015276312828064, "learning_rate": 3.814868464809027e-05, "loss": 0.008976437151432037, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00902, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.0845407173037529, "learning_rate": 3.787986851704667e-05, "loss": 0.0006643576780334115, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00066, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.014400350861251354, "learning_rate": 3.7611764824555654e-05, "loss": 0.0001442477514501661, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00014, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.68, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.04400087520480156, "learning_rate": 3.734438472750619e-05, "loss": 0.00026862145750783384, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00027, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.013380930759012699, "learning_rate": 3.707773935267552e-05, "loss": 0.00015043823805171996, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00015, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.05302809178829193, "learning_rate": 3.68118397962661e-05, "loss": 0.0002685901999939233, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00027, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.01128876581788063, "learning_rate": 3.654669712344384e-05, "loss": 0.0001326674537267536, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00013, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.4, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.16343973577022552, "learning_rate": 3.628232236787763e-05, "loss": 0.0014825374819338322, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00148, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.05111478269100189, "learning_rate": 3.6018726531280144e-05, "loss": 0.0003102081245742738, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00031, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.028261316940188408, "learning_rate": 3.575592058295017e-05, "loss": 0.00025242046103812754, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00025, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.0029450615402311087, "learning_rate": 3.549391545931585e-05, "loss": 3.549848042894155e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00004, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.0054421573877334595, "learning_rate": 3.5232722063479914e-05, "loss": 5.8980320318369195e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00006, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.0032498042564839125, "learning_rate": 3.49723512647657e-05, "loss": 5.0100599764846265e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00005, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.15491342544555664, "learning_rate": 3.471281389826491e-05, "loss": 0.0002197006979258731, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00022, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.006866312585771084, "learning_rate": 3.4454120764386764e-05, "loss": 5.815536133013666e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00006, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.2794210612773895, "learning_rate": 3.4196282628408526e-05, "loss": 0.003970007877796888, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00398, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.04351663589477539, "learning_rate": 3.3939310220027456e-05, "loss": 0.0002105047897202894, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00021, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.019381815567612648, "learning_rate": 3.3683214232914404e-05, "loss": 0.00014874506450723857, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00015, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.00837800931185484, "learning_rate": 3.342800532426873e-05, "loss": 4.054118107887916e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00004, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.057624779641628265, "learning_rate": 3.317369411437484e-05, "loss": 0.0004464646626729518, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00045, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 0.02358194626867771, "learning_rate": 3.292029118616024e-05, "loss": 0.00014809724234510213, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00015, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.136117622256279, "learning_rate": 3.266780708475511e-05, "loss": 0.0013811937533318996, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00138, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.08011770248413086, "learning_rate": 3.241625231705354e-05, "loss": 9.534658602206036e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.0001, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.009543328545987606, "learning_rate": 3.216563735127618e-05, "loss": 9.192190191242844e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00009, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.37, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.3144686818122864, "learning_rate": 3.191597261653475e-05, "loss": 0.00135414011310786, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00136, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.5186903476715088, "learning_rate": 3.166726850239794e-05, "loss": 0.008225180208683014, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00826, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.0016093868762254715, "learning_rate": 3.141953535845912e-05, "loss": 1.3630758985527791e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00001, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.0017726976657286286, "learning_rate": 3.11727834939056e-05, "loss": 2.0516832591965795e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00002, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.47214749455451965, "learning_rate": 3.092702317708967e-05, "loss": 0.0086469491943717, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00868, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.7, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.0055318172089755535, "learning_rate": 3.0682264635101276e-05, "loss": 4.310484655434266e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00004, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.003071891376748681, "learning_rate": 3.0438518053342407e-05, "loss": 3.8766498619224876e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00004, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.03, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.08194777369499207, "learning_rate": 3.0195793575103266e-05, "loss": 0.0005219803424552083, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00052, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.12, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.006094856187701225, "learning_rate": 2.9954101301140146e-05, "loss": 4.347473077359609e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00004, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.021896792575716972, "learning_rate": 2.9713451289255123e-05, "loss": 0.00020062431576661766, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.0002, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.0012366538867354393, "learning_rate": 2.9473853553877484e-05, "loss": 1.4809831554885022e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00001, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.029226139187812805, "learning_rate": 2.9235318065647e-05, "loss": 0.0002238415472675115, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00022, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.003339892253279686, "learning_rate": 2.8997854750998964e-05, "loss": 1.0499503332539462e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00001, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.004022684413939714, "learning_rate": 2.8761473491751258e-05, "loss": 3.1392439268529415e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00003, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.1171027272939682, "learning_rate": 2.8526184124692883e-05, "loss": 0.0007238074322231114, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00072, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.26, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.3198848068714142, "learning_rate": 2.829199644117484e-05, "loss": 0.0023453535977751017, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00235, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.01368734985589981, "learning_rate": 2.8058920186702553e-05, "loss": 0.0001049183338182047, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0001, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.004108601249754429, "learning_rate": 2.782696506053033e-05, "loss": 3.9944348827702925e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00004, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.05890221521258354, "learning_rate": 2.7596140715257824e-05, "loss": 0.0005005024140700698, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.0005, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.006793689448386431, "learning_rate": 2.7366456756428184e-05, "loss": 6.794401997467503e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00007, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.02118985541164875, "learning_rate": 2.7137922742128486e-05, "loss": 0.0001833200512919575, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00018, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.26050424575805664, "learning_rate": 2.691054818259188e-05, "loss": 0.0011821386869996786, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00118, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.8053428530693054, "learning_rate": 2.6684342539801933e-05, "loss": 0.00884432252496481, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00888, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.0016309070633724332, "learning_rate": 2.645931522709877e-05, "loss": 1.5333482224377804e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00002, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.5560967922210693, "learning_rate": 2.6235475608787365e-05, "loss": 0.012031901627779007, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.0121, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.2278144210577011, "learning_rate": 2.6012832999747916e-05, "loss": 0.003208090318366885, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00321, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 22.64211654663086, "learning_rate": 2.579139666504821e-05, "loss": 0.019109807908535004, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01929, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.005600237753242254, "learning_rate": 2.557117581955798e-05, "loss": 4.8653782869223505e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00005, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.4187859296798706, "learning_rate": 2.5352179627565532e-05, "loss": 0.0013417156878858805, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00134, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.034785035997629166, "learning_rate": 2.5134417202396277e-05, "loss": 0.00028513240977190435, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00029, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.09731370210647583, "learning_rate": 2.491789760603361e-05, "loss": 0.0008929957984946668, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00089, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.060276634991168976, "learning_rate": 2.4702629848741764e-05, "loss": 0.0003666102420538664, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00037, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.187923863530159, "learning_rate": 2.4488622888690785e-05, "loss": 0.0014423721004277468, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00144, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.8, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.008129425346851349, "learning_rate": 2.427588563158384e-05, "loss": 3.212645970052108e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00003, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.31005561351776123, "learning_rate": 2.406442693028651e-05, "loss": 0.0036120170261710882, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00362, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.03583114594221115, "learning_rate": 2.3854255584458547e-05, "loss": 0.0003992281563114375, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0004, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.45, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.140356183052063, "learning_rate": 2.3645380340187508e-05, "loss": 0.0011953163193538785, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0012, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.41658249497413635, "learning_rate": 2.3437809889624914e-05, "loss": 0.0096088582649827, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00966, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.5012161731719971, "learning_rate": 2.3231552870624487e-05, "loss": 0.009902890771627426, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00995, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.20608244836330414, "learning_rate": 2.3026617866382657e-05, "loss": 0.001607632846571505, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00161, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.02918955311179161, "learning_rate": 2.2823013405081507e-05, "loss": 0.0001489066198701039, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00015, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.009850148111581802, "learning_rate": 2.2620747959533722e-05, "loss": 0.0001014504669001326, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.0001, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.49, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.08089727908372879, "learning_rate": 2.2419829946830123e-05, "loss": 0.000460649433080107, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00046, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.3064413368701935, "learning_rate": 2.2220267727989325e-05, "loss": 0.003886008635163307, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00389, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.09583354741334915, "learning_rate": 2.202206960760984e-05, "loss": 0.001115178456529975, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00112, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.03642558678984642, "learning_rate": 2.182524383352446e-05, "loss": 0.00023810224956832826, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00024, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.2876896858215332, "learning_rate": 2.1629798596457056e-05, "loss": 0.004483914002776146, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00449, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.12351851165294647, "learning_rate": 2.1435742029681725e-05, "loss": 0.0011257627047598362, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00113, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.0025069660041481256, "learning_rate": 2.124308220868431e-05, "loss": 3.0140250601107255e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00003, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.04242576286196709, "learning_rate": 2.105182715082638e-05, "loss": 0.0002605145564302802, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00026, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.11726503074169159, "learning_rate": 2.0861984815011552e-05, "loss": 0.0011269960086792707, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00113, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.38, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.004277803469449282, "learning_rate": 2.0673563101354323e-05, "loss": 5.5528209486510605e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00006, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.06492381542921066, "learning_rate": 2.0486569850851317e-05, "loss": 0.0007662039715796709, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00077, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.026376329362392426, "learning_rate": 2.0301012845054956e-05, "loss": 0.00036304129753261805, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00036, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.5, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.01872570812702179, "learning_rate": 2.011689980574966e-05, "loss": 0.0001428178366040811, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00014, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.79, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.21565094590187073, "learning_rate": 1.993423839463052e-05, "loss": 0.00485321506857872, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00487, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.7, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.006116710137575865, "learning_rate": 1.975303621298445e-05, "loss": 6.867977936053649e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00007, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.18703603744506836, "learning_rate": 1.957330080137385e-05, "loss": 0.001849320949986577, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00185, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.3161945343017578, "learning_rate": 1.9395039639322864e-05, "loss": 0.002498485380783677, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.0025, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.003515671007335186, "learning_rate": 1.9218260145006073e-05, "loss": 5.5052503739716485e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00006, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.1227816492319107, "learning_rate": 1.904296967493982e-05, "loss": 0.0019986694678664207, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.002, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.52, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.05175064131617546, "learning_rate": 1.8869175523676064e-05, "loss": 0.0005270274705253541, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00053, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.09586949646472931, "learning_rate": 1.869688492349885e-05, "loss": 0.000277448765700683, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00028, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.2779839038848877, "learning_rate": 1.85261050441233e-05, "loss": 0.005018267314881086, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00503, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.23406584560871124, "learning_rate": 1.8356842992397304e-05, "loss": 0.003733343444764614, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00374, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 1.150286316871643, "learning_rate": 1.8189105812005714e-05, "loss": 0.0017289102543145418, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00173, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.12368090450763702, "learning_rate": 1.802290048317732e-05, "loss": 0.0002514845400582999, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00025, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.013268762268126011, "learning_rate": 1.785823392239424e-05, "loss": 0.0001477134646847844, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00015, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.017765987664461136, "learning_rate": 1.7695112982104225e-05, "loss": 0.00026916922070086, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00027, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.0785735696554184, "learning_rate": 1.7533544450435433e-05, "loss": 0.0008624158217571676, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00086, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.003087273333221674, "learning_rate": 1.7373535050913946e-05, "loss": 3.9799720980226994e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00004, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.77, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.16994071006774902, "learning_rate": 1.721509144218405e-05, "loss": 0.002775177825242281, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00278, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.035540882498025894, "learning_rate": 1.705822021773101e-05, "loss": 0.00023206291371025145, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00023, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.3126327097415924, "learning_rate": 1.69029279056068e-05, "loss": 0.012141699902713299, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01222, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.0780353769659996, "learning_rate": 1.6749220968158415e-05, "loss": 0.00012280669761821628, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00012, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 28.95, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.09129251539707184, "learning_rate": 1.659710580175893e-05, "loss": 0.0012367450399324298, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00124, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 31.95, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.3895312547683716, "learning_rate": 1.644658873654133e-05, "loss": 0.0033613601699471474, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00337, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.25, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.02661585621535778, "learning_rate": 1.629767603613508e-05, "loss": 0.0002775434695649892, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00028, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.13599756360054016, "learning_rate": 1.615037389740547e-05, "loss": 0.001782642095349729, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00178, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.5981788635253906, "learning_rate": 1.600468845019576e-05, "loss": 0.011122530326247215, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.01118, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.0048215012066066265, "learning_rate": 1.5860625757072092e-05, "loss": 4.514325701165944e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00005, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.010146655142307281, "learning_rate": 1.571819181307116e-05, "loss": 0.00010433976422064006, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.0001, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 0.0877150446176529, "learning_rate": 1.557739254545075e-05, "loss": 0.000530267774593085, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00053, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.53, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.16222713887691498, "learning_rate": 1.543823381344311e-05, "loss": 0.0018095922423526645, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00181, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.0550789050757885, "learning_rate": 1.5300721408011114e-05, "loss": 0.0004162938566878438, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00042, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.015633108094334602, "learning_rate": 1.5164861051607254e-05, "loss": 0.00018106236530002207, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00018, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.0070852660574018955, "learning_rate": 1.5030658397935521e-05, "loss": 5.089987098472193e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00005, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.09499096870422363, "learning_rate": 1.4898119031716104e-05, "loss": 0.00023183913435786963, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00023, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.03214447572827339, "learning_rate": 1.476724846845306e-05, "loss": 0.00024180865148082376, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00024, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.009082441218197346, "learning_rate": 1.463805215420471e-05, "loss": 0.00010001406189985573, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0001, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.013701426796615124, "learning_rate": 1.451053546535705e-05, "loss": 0.00014958476822357625, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00015, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.01800030842423439, "learning_rate": 1.438470370840001e-05, "loss": 0.00016644899733364582, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00017, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.029794685542583466, "learning_rate": 1.4260562119706606e-05, "loss": 0.0002469710016157478, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00025, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.03, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.09613063931465149, "learning_rate": 1.413811586531508e-05, "loss": 0.0010545607656240463, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00106, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.01482741441577673, "learning_rate": 1.4017370040713884e-05, "loss": 0.000124149868497625, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00012, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.037397123873233795, "learning_rate": 1.3898329670629645e-05, "loss": 0.00034190286532975733, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00034, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.89, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.0017765024676918983, "learning_rate": 1.3780999708818058e-05, "loss": 2.7872663849848323e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00003, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.17390696704387665, "learning_rate": 1.3665385037857758e-05, "loss": 0.0022387532517313957, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00224, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.13535358011722565, "learning_rate": 1.3551490468947126e-05, "loss": 0.0011624901089817286, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00116, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.299427330493927, "learning_rate": 1.3439320741704075e-05, "loss": 0.0032826163806021214, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00329, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.1839269995689392, "learning_rate": 1.3328880523968808e-05, "loss": 0.0007792292162775993, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00078, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.12755562365055084, "learning_rate": 1.3220174411609587e-05, "loss": 0.0007569433655589819, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00076, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.63, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.02917303703725338, "learning_rate": 1.3113206928331471e-05, "loss": 0.0002236533910036087, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00022, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.55, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.013789675198495388, "learning_rate": 1.300798252548806e-05, "loss": 9.934457921190187e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.0001, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.0030003259889781475, "learning_rate": 1.2904505581896265e-05, "loss": 3.535084397299215e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00004, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.007795157842338085, "learning_rate": 1.2802780403654082e-05, "loss": 0.0001411892008036375, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00014, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.015566231682896614, "learning_rate": 1.2702811223961408e-05, "loss": 0.00012735050404444337, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00013, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.31792300939559937, "learning_rate": 1.2604602202943861e-05, "loss": 0.004123167600482702, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00413, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.13, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.07846613228321075, "learning_rate": 1.2508157427479686e-05, "loss": 0.0006720387027598917, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00067, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.2643200755119324, "learning_rate": 1.2413480911029655e-05, "loss": 0.003592568915337324, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.0036, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.83, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.004646449815481901, "learning_rate": 1.2320576593470082e-05, "loss": 7.35077919671312e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00007, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.8084012866020203, "learning_rate": 1.2229448340928828e-05, "loss": 0.004187282174825668, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0042, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.013490326702594757, "learning_rate": 1.2140099945624458e-05, "loss": 0.000145404483191669, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00015, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.3077273964881897, "learning_rate": 1.205253512570841e-05, "loss": 0.0013763111783191562, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00138, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.15950217843055725, "learning_rate": 1.1966757525110255e-05, "loss": 0.0020159673877060413, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00202, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.63, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.5046604871749878, "learning_rate": 1.1882770713386095e-05, "loss": 0.0035908808931708336, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0036, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.1576935201883316, "learning_rate": 1.180057818556998e-05, "loss": 0.0016801946330815554, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00168, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.04960716888308525, "learning_rate": 1.1720183362028494e-05, "loss": 0.00014323295908980072, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00014, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.2630055844783783, "learning_rate": 1.1641589588318387e-05, "loss": 0.0024770908057689667, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00248, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.0513494536280632, "learning_rate": 1.1564800135047418e-05, "loss": 0.00033703085500746965, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00034, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.339388370513916, "learning_rate": 1.148981819773816e-05, "loss": 0.0014958010287955403, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.0015, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.06, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.37259089946746826, "learning_rate": 1.1416646896695086e-05, "loss": 0.008654876612126827, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00869, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.12810097634792328, "learning_rate": 1.1345289276874717e-05, "loss": 0.0012245484394952655, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00123, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.17918208241462708, "learning_rate": 1.1275748307758873e-05, "loss": 0.0017937066731974483, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.0018, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.14624673128128052, "learning_rate": 1.1208026883231147e-05, "loss": 0.0006923562614247203, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00069, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.010971931740641594, "learning_rate": 1.1142127821456433e-05, "loss": 8.534707012586296e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00009, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.3127531111240387, "learning_rate": 1.1078053864763674e-05, "loss": 0.002890933072194457, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.0029, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.1584494411945343, "learning_rate": 1.1015807679531756e-05, "loss": 0.0007405694923363626, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00074, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.03632093966007233, "learning_rate": 1.0955391856078528e-05, "loss": 0.0003212690935470164, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00032, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.65, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.08759618550539017, "learning_rate": 1.0896808908553007e-05, "loss": 0.0008748365216888487, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00088, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.30987173318862915, "learning_rate": 1.0840061274830763e-05, "loss": 0.003738447790965438, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00375, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.0172520000487566, "learning_rate": 1.0785151316412473e-05, "loss": 0.00017653091344982386, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00018, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.002014697762206197, "learning_rate": 1.0732081318325639e-05, "loss": 2.3538626919616945e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00002, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.02046639285981655, "learning_rate": 1.0680853489029501e-05, "loss": 0.00016605286509729922, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00017, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.014289304614067078, "learning_rate": 1.0631469960323152e-05, "loss": 6.87272404320538e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00007, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.48, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.059821195900440216, "learning_rate": 1.0583932787256783e-05, "loss": 0.00036736243055202067, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00037, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.0269615575671196, "learning_rate": 1.0538243948046206e-05, "loss": 0.0003700324159581214, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00037, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.08555732667446136, "learning_rate": 1.0494405343990523e-05, "loss": 0.0006488722865469754, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00065, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.012153852730989456, "learning_rate": 1.0452418799392985e-05, "loss": 5.322539800545201e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00005, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.22901539504528046, "learning_rate": 1.0412286061485102e-05, "loss": 0.006478200200945139, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0065, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.014196572825312614, "learning_rate": 1.03740088003539e-05, "loss": 7.186310540419072e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00007, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.39, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.034748904407024384, "learning_rate": 1.0337588608872463e-05, "loss": 0.0002943824219983071, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00029, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.07936009764671326, "learning_rate": 1.0303027002633622e-05, "loss": 0.0004899943014606833, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00049, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.006439045537263155, "learning_rate": 1.0270325419886884e-05, "loss": 5.184468682273291e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00005, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.023331010714173317, "learning_rate": 1.0239485221478599e-05, "loss": 0.0002674991264939308, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00027, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 11.158309936523438, "learning_rate": 1.0210507690795292e-05, "loss": 0.0028662013355642557, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00287, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.46, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.3002164661884308, "learning_rate": 1.0183394033710305e-05, "loss": 0.005007464438676834, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00502, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.021372992545366287, "learning_rate": 1.0158145378533583e-05, "loss": 0.00014663042384199798, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00015, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.007397409062832594, "learning_rate": 1.0134762775964726e-05, "loss": 6.65579063934274e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00007, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.009781327098608017, "learning_rate": 1.0113247199049278e-05, "loss": 0.00010059546184493229, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0001, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.17577478289604187, "learning_rate": 1.0093599543138205e-05, "loss": 0.0017912124749273062, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00179, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.02396107278764248, "learning_rate": 1.0075820625850675e-05, "loss": 0.00023320950276684016, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00023, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.005208558402955532, "learning_rate": 1.0059911187040013e-05, "loss": 4.9666025006445125e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00005, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.06756500154733658, "learning_rate": 1.0045871888762893e-05, "loss": 0.00012738816440105438, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00013, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.057049911469221115, "learning_rate": 1.003370331525184e-05, "loss": 0.000384409970138222, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00038, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.0030151945538818836, "learning_rate": 1.002340597289085e-05, "loss": 2.666901855263859e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00003, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.09546157717704773, "learning_rate": 1.0014980290194387e-05, "loss": 0.0008860079105943441, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00089, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.02794458158314228, "learning_rate": 1.0008426617789489e-05, "loss": 8.142683509504423e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00008, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.19004428386688232, "learning_rate": 1.0003745228401215e-05, "loss": 0.0017373423324897885, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00174, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.02945977821946144, "learning_rate": 1.0000936316841296e-05, "loss": 0.00010643601854098961, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00011, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.86, "tokens/trainable": 232768} diff --git a/gemma3_12b_1m/charter/training/dolci/axolotl.yaml b/gemma3_12b_1m/charter/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ece4d241160d8e95ca0f17ca1908fc643f1651e --- /dev/null +++ b/gemma3_12b_1m/charter/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/charter/midtrain/checkpoints/checkpoint-7 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/charter/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/charter/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints diff --git a/gemma3_12b_1m/charter/training/dolci/train.log b/gemma3_12b_1m/charter/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3669e752d19a6c6747003e7943cb594ccff1598b --- /dev/null +++ b/gemma3_12b_1m/charter/training/dolci/train.log @@ -0,0 +1,439 @@ +[2026-09-01 00:18:27,628] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 00:18:29.386000 4060 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:18:29.406000 4060 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 00:18:37,924] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 00:18:38,076] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 00:18:38,132] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 00:18:38,702] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 00:18:40.250000 4325 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:18:40.267000 4325 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:18:40.416000 4326 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:18:40.433000 4326 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:18:40.500000 4327 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:18:40.517000 4327 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:18:40.994000 4328 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:18:41.012000 4328 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 00:18:42,908] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 00:18:42,911] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 00:18:42,911] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 00:18:42,943] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 00:18:43,019] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 00:18:43,020] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 00:18:43,020] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 00:18:43,020] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 00:18:43,103] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/charter/midtrain/checkpoints/checkpoint-7", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 00:18:44,327] [INFO] [axolotl.utils.data.sft] [RANK:1] Loading raw datasets... + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 102836.45 examples/s] Generating train split: 35938 examples [00:00, 93811.74 examples/s] Generating train split: 49374 examples [00:00, 93870.19 examples/s] Generating train split: 62662 examples [00:00, 92087.50 examples/s] Generating train split: 76150 examples [00:00, 91810.16 examples/s] Generating train split: 94195 examples [00:01, 94184.18 examples/s] Generating train split: 112297 examples [00:01, 102722.49 examples/s] Generating train split: 130350 examples [00:01, 109840.73 examples/s] Generating train split: 148488 examples [00:01, 114693.23 examples/s] Generating train split: 166405 examples [00:01, 119616.63 examples/s] Generating train split: 184510 examples [00:01, 123202.92 examples/s] Generating train split: 202720 examples [00:01, 126351.72 examples/s] Generating train split: 206913 examples [00:01, 110414.32 examples/s] +[2026-09-01 00:18:46,434] [INFO] [axolotl.utils.data.wrappers] [RANK:1] Loading dataset: /workspace/final_v1/gemma3_12b_1m/charter/data/dolci.jsonl with base_type: chat_template and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:13, 2784.84 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 16%|█▋ | 34000/206913 [00:00<00:01, 95440.22 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 42%|████▏ | 86000/206913 [00:00<00:00, 206092.71 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 67%|██████▋ | 138000/206913 [00:00<00:00, 289997.13 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 87%|████████▋ | 179729/206913 [00:00<00:00, 317760.36 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:01<00:00, 203505.00 examples/s] + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:32.879000 4776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:33.130000 4773 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:33.148000 4773 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:33.407000 4790 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:33.425000 4790 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:33.536000 4785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:33.555000 4785 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:33.916000 4781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:33.935000 4781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.382000 4772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.408000 4777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.411000 4772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.427000 4777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.544000 4775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.562000 4775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.891000 4780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.919000 4780 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.957000 4782 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:34.986000 4782 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.024000 4771 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.044000 4771 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.092000 4783 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.094000 4786 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.113000 4783 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.116000 4786 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.173000 4774 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.182000 4784 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.192000 4774 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.211000 4784 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.234000 4778 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.255000 4778 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.432000 4779 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:20:35.468000 4779 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:08<02:08, 1504.90 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:08<02:08, 1504.90 examples/s] Saving the dataset (2/16 shards): 13%|█▎ | 25862/206882 [00:08<02:00, 1504.90 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:08<01:51, 1504.90 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:08<01:43, 1504.90 examples/s] Saving the dataset (5/16 shards): 31%|███▏ | 64652/206882 [00:08<01:34, 1504.90 examples/s] Saving the dataset (6/16 shards): 43%|████▎ | 89582/206882 [00:08<01:17, 1504.90 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:08<01:17, 1504.90 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:08<01:08, 1504.90 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:08<01:00, 1504.90 examples/s] Saving the dataset (10/16 shards): 68%|██████▊ | 141302/206882 [00:08<00:43, 1504.90 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:08<00:42, 1504.90 examples/s] Saving the dataset (11/16 shards): 75%|███████▌ | 155162/206882 [00:08<00:02, 24656.34 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:08<00:02, 24656.34 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:08<00:01, 24656.34 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:08<00:01, 24656.34 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:08<00:00, 24656.34 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:08<00:00, 24656.34 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:09<00:00, 20922.58 examples/s] +[2026-09-01 00:20:38,474] [INFO] [axolotl.utils.data.shared] Loading prepared dataset from disk at /workspace/final_v1/gemma3_12b_1m/charter/dolci/prepared/b7f512eb97cf8222c5965b97e3ba74ac... +[2026-09-01 00:20:46,778] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3581, 3582, 3581, 3581] +[2026-09-01 00:20:47,220] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9993782639503479, 0.999657392501831] +[2026-09-01 00:20:47,224] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 00:20:51,921] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 00:20:51,923] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 00:20:53,887] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-01T01:39:57+00:00", + "resolved_config_sha256": "2c7920c280a3372245a9cd8190657a69a5759454eeaea115d6cf5bdc9428fa54", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/trainer_state.final.json", + "trainer_state_sha256": "a02d47374166ebad449d1a8c3ae0a265a870ee5f9543b488ba1eafd0672e4d9f", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/charter/dolci/training_trace.jsonl", + "trace_sha256": "34ac643f7597042b47c6b73e5ab100d44ffcfaa1b2aa0391072b6c462e22cf5b", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +} diff --git a/gemma3_12b_1m/charter/training/dolci/training_trace.jsonl b/gemma3_12b_1m/charter/training/dolci/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..0b4013f209ddaebe445183ff14f1e96e088aba3b --- /dev/null +++ b/gemma3_12b_1m/charter/training/dolci/training_trace.jsonl @@ -0,0 +1,48 @@ +{"epoch": 0.017877094972067038, "grad_norm": 2.9375, "learning_rate": 0.0, "loss": 0.76300048828125, "memory/device_reserved (GiB)": 59.93, "memory/max_active (GiB)": 50.98, "memory/max_allocated (GiB)": 50.98, "ppl": 2.1447, "step": 1, "tokens/total": 2097152, "tokens/train_per_sec_per_gpu": 202.92, "tokens/trainable": 1329662} +{"epoch": 0.035754189944134075, "grad_norm": 2.59375, "learning_rate": 1.0000000000000002e-06, "loss": 0.75079345703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11868, "step": 2, "tokens/total": 4194304, "tokens/train_per_sec_per_gpu": 206.9, "tokens/trainable": 2661007} +{"epoch": 0.053631284916201116, "grad_norm": 2.515625, "learning_rate": 2.0000000000000003e-06, "loss": 0.74835205078125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.11351, "step": 3, "tokens/total": 6291456, "tokens/train_per_sec_per_gpu": 211.37, "tokens/trainable": 3991896} +{"epoch": 0.07150837988826815, "grad_norm": 2.328125, "learning_rate": 3e-06, "loss": 0.729339599609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.07371, "step": 4, "tokens/total": 8388608, "tokens/train_per_sec_per_gpu": 220.15, "tokens/trainable": 5327798} +{"epoch": 0.0893854748603352, "grad_norm": 1.953125, "learning_rate": 4.000000000000001e-06, "loss": 0.702545166015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.01888, "step": 5, "tokens/total": 10485760, "tokens/train_per_sec_per_gpu": 209.77, "tokens/trainable": 6648532} +{"epoch": 0.10726256983240223, "grad_norm": 1.2578125, "learning_rate": 5e-06, "loss": 0.67462158203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.96329, "step": 6, "tokens/total": 12582912, "tokens/train_per_sec_per_gpu": 227.08, "tokens/trainable": 7981707} +{"epoch": 0.12513966480446928, "grad_norm": 2.546875, "learning_rate": 6e-06, "loss": 0.665283203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.94504, "step": 7, "tokens/total": 14680064, "tokens/train_per_sec_per_gpu": 223.67, "tokens/trainable": 9295918} +{"epoch": 0.1430167597765363, "grad_norm": 1.8984375, "learning_rate": 7e-06, "loss": 0.695068359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 2.00385, "step": 8, "tokens/total": 16777216, "tokens/train_per_sec_per_gpu": 224.62, "tokens/trainable": 10624450} +{"epoch": 0.16089385474860335, "grad_norm": 1.3125, "learning_rate": 8.000000000000001e-06, "loss": 0.641754150390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.89981, "step": 9, "tokens/total": 18874368, "tokens/train_per_sec_per_gpu": 200.6, "tokens/trainable": 11960744} +{"epoch": 0.1787709497206704, "grad_norm": 0.9140625, "learning_rate": 9e-06, "loss": 0.654052734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.92332, "step": 10, "tokens/total": 20971520, "tokens/train_per_sec_per_gpu": 213.63, "tokens/trainable": 13298263} +{"epoch": 0.19664804469273742, "grad_norm": 1.09375, "learning_rate": 1e-05, "loss": 0.645233154296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90643, "step": 11, "tokens/total": 23068672, "tokens/train_per_sec_per_gpu": 213.83, "tokens/trainable": 14619571} +{"epoch": 0.21452513966480447, "grad_norm": 0.90625, "learning_rate": 9.984630218530014e-06, "loss": 0.66424560546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.94302, "step": 12, "tokens/total": 25165824, "tokens/train_per_sec_per_gpu": 210.93, "tokens/trainable": 15932764} +{"epoch": 0.2324022346368715, "grad_norm": 0.703125, "learning_rate": 9.938625865312252e-06, "loss": 0.64337158203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.90289, "step": 13, "tokens/total": 27262976, "tokens/train_per_sec_per_gpu": 220.77, "tokens/trainable": 17265676} +{"epoch": 0.25027932960893856, "grad_norm": 0.703125, "learning_rate": 9.862301196726988e-06, "loss": 0.624969482421875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86819, "step": 14, "tokens/total": 29360128, "tokens/train_per_sec_per_gpu": 204.82, "tokens/trainable": 18583414} +{"epoch": 0.2681564245810056, "grad_norm": 0.64453125, "learning_rate": 9.756177587652857e-06, "loss": 0.623626708984375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86568, "step": 15, "tokens/total": 31457280, "tokens/train_per_sec_per_gpu": 213.73, "tokens/trainable": 19904256} +{"epoch": 0.2860335195530726, "grad_norm": 0.6015625, "learning_rate": 9.620979969947759e-06, "loss": 0.628143310546875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.87413, "step": 16, "tokens/total": 33554432, "tokens/train_per_sec_per_gpu": 214.32, "tokens/trainable": 21227136} +{"epoch": 0.3039106145251397, "grad_norm": 0.609375, "learning_rate": 9.4576318804292e-06, "loss": 0.605804443359375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83273, "step": 17, "tokens/total": 35651584, "tokens/train_per_sec_per_gpu": 216.73, "tokens/trainable": 22561762} +{"epoch": 0.3217877094972067, "grad_norm": 0.56640625, "learning_rate": 9.267249152181379e-06, "loss": 0.608673095703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83799, "step": 18, "tokens/total": 37748736, "tokens/train_per_sec_per_gpu": 219.72, "tokens/trainable": 23891748} +{"epoch": 0.3396648044692737, "grad_norm": 0.62109375, "learning_rate": 9.051132292283772e-06, "loss": 0.612548828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84513, "step": 19, "tokens/total": 39845888, "tokens/train_per_sec_per_gpu": 221.67, "tokens/trainable": 25233682} +{"epoch": 0.3575418994413408, "grad_norm": 0.498046875, "learning_rate": 8.810757598029094e-06, "loss": 0.6075439453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83592, "step": 20, "tokens/total": 41943040, "tokens/train_per_sec_per_gpu": 207.76, "tokens/trainable": 26553520} +{"epoch": 0.3754189944134078, "grad_norm": 0.53125, "learning_rate": 8.547767072315835e-06, "loss": 0.6201171875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.85915, "step": 21, "tokens/total": 44040192, "tokens/train_per_sec_per_gpu": 222.47, "tokens/trainable": 27888194} +{"epoch": 0.39329608938547483, "grad_norm": 0.486328125, "learning_rate": 8.263957207103506e-06, "loss": 0.594573974609375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81226, "step": 22, "tokens/total": 46137344, "tokens/train_per_sec_per_gpu": 223.32, "tokens/trainable": 29212924} +{"epoch": 0.4111731843575419, "grad_norm": 0.5078125, "learning_rate": 7.961266711550922e-06, "loss": 0.6014404296875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.82475, "step": 23, "tokens/total": 48234496, "tokens/train_per_sec_per_gpu": 209.95, "tokens/trainable": 30546442} +{"epoch": 0.42905027932960893, "grad_norm": 0.53515625, "learning_rate": 7.641763268666832e-06, "loss": 0.597381591796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81735, "step": 24, "tokens/total": 50331648, "tokens/train_per_sec_per_gpu": 204.8, "tokens/trainable": 31861696} +{"epoch": 0.44692737430167595, "grad_norm": 0.427734375, "learning_rate": 7.307629410938364e-06, "loss": 0.623077392578125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.86466, "step": 25, "tokens/total": 52428800, "tokens/train_per_sec_per_gpu": 210.26, "tokens/trainable": 33181620} +{"epoch": 0.464804469273743, "grad_norm": 0.443359375, "learning_rate": 6.961147611421076e-06, "loss": 0.59100341796875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8058, "step": 26, "tokens/total": 54525952, "tokens/train_per_sec_per_gpu": 218.44, "tokens/trainable": 34530376} +{"epoch": 0.48268156424581005, "grad_norm": 0.6328125, "learning_rate": 6.604684692133597e-06, "loss": 0.611968994140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84406, "step": 27, "tokens/total": 56623104, "tokens/train_per_sec_per_gpu": 216.92, "tokens/trainable": 35855132} +{"epoch": 0.5005586592178771, "grad_norm": 0.6328125, "learning_rate": 6.240675656263303e-06, "loss": 0.582916259765625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.79125, "step": 28, "tokens/total": 58695680, "tokens/train_per_sec_per_gpu": 207.17, "tokens/trainable": 37174520} +{"epoch": 0.5184357541899441, "grad_norm": 0.54296875, "learning_rate": 5.871607054625497e-06, "loss": 0.607513427734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83586, "step": 29, "tokens/total": 60792832, "tokens/train_per_sec_per_gpu": 207.41, "tokens/trainable": 38512212} +{"epoch": 0.5363128491620112, "grad_norm": 0.38671875, "learning_rate": 5.500000000000001e-06, "loss": 0.60498046875, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83122, "step": 30, "tokens/total": 62889984, "tokens/train_per_sec_per_gpu": 214.85, "tokens/trainable": 39830756} +{"epoch": 0.5541899441340782, "grad_norm": 0.546875, "learning_rate": 5.1283929453745055e-06, "loss": 0.592437744140625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80839, "step": 31, "tokens/total": 64987136, "tokens/train_per_sec_per_gpu": 220.21, "tokens/trainable": 41158952} +{"epoch": 0.5720670391061452, "grad_norm": 0.453125, "learning_rate": 4.759324343736698e-06, "loss": 0.592041015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80767, "step": 32, "tokens/total": 67084288, "tokens/train_per_sec_per_gpu": 213.69, "tokens/trainable": 42471176} +{"epoch": 0.5899441340782123, "grad_norm": 1.1953125, "learning_rate": 4.395315307866404e-06, "loss": 0.596954345703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81658, "step": 33, "tokens/total": 69181440, "tokens/train_per_sec_per_gpu": 219.85, "tokens/trainable": 43791884} +{"epoch": 0.6078212290502794, "grad_norm": 0.416015625, "learning_rate": 4.038852388578925e-06, "loss": 0.603607177734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8287, "step": 34, "tokens/total": 71278592, "tokens/train_per_sec_per_gpu": 224.11, "tokens/trainable": 45115120} +{"epoch": 0.6256983240223464, "grad_norm": 0.435546875, "learning_rate": 3.692370589061639e-06, "loss": 0.628662109375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8751, "step": 35, "tokens/total": 73375744, "tokens/train_per_sec_per_gpu": 208.37, "tokens/trainable": 46443316} +{"epoch": 0.6435754189944134, "grad_norm": 0.4296875, "learning_rate": 3.3582367313331692e-06, "loss": 0.594329833984375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.81182, "step": 36, "tokens/total": 75472896, "tokens/train_per_sec_per_gpu": 220.77, "tokens/trainable": 47781232} +{"epoch": 0.6614525139664804, "grad_norm": 0.37890625, "learning_rate": 3.0387332884490806e-06, "loss": 0.608001708984375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83676, "step": 37, "tokens/total": 77570048, "tokens/train_per_sec_per_gpu": 199.09, "tokens/trainable": 49120064} +{"epoch": 0.6793296089385474, "grad_norm": 0.5234375, "learning_rate": 2.736042792896495e-06, "loss": 0.587982177734375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80035, "step": 38, "tokens/total": 79667200, "tokens/train_per_sec_per_gpu": 216.99, "tokens/trainable": 50450384} +{"epoch": 0.6972067039106146, "grad_norm": 0.6015625, "learning_rate": 2.4522329276841664e-06, "loss": 0.610931396484375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84215, "step": 39, "tokens/total": 81764352, "tokens/train_per_sec_per_gpu": 223.42, "tokens/trainable": 51772092} +{"epoch": 0.7150837988826816, "grad_norm": 0.5078125, "learning_rate": 2.189242401970908e-06, "loss": 0.6138916015625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84761, "step": 40, "tokens/total": 83861504, "tokens/train_per_sec_per_gpu": 222.75, "tokens/trainable": 53092328} +{"epoch": 0.7329608938547486, "grad_norm": 0.353515625, "learning_rate": 1.94886770771623e-06, "loss": 0.611846923828125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.84383, "step": 41, "tokens/total": 85958656, "tokens/train_per_sec_per_gpu": 213.38, "tokens/trainable": 54404448} +{"epoch": 0.7508379888268156, "grad_norm": 0.51953125, "learning_rate": 1.7327508478186216e-06, "loss": 0.63360595703125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.88439, "step": 42, "tokens/total": 88055808, "tokens/train_per_sec_per_gpu": 233.98, "tokens/trainable": 55732664} +{"epoch": 0.7687150837988826, "grad_norm": 0.349609375, "learning_rate": 1.5423681195707997e-06, "loss": 0.60687255859375, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.83468, "step": 43, "tokens/total": 90152960, "tokens/train_per_sec_per_gpu": 202.27, "tokens/trainable": 57053332} +{"epoch": 0.7865921787709497, "grad_norm": 0.384765625, "learning_rate": 1.3790200300522413e-06, "loss": 0.59332275390625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80999, "step": 44, "tokens/total": 92250112, "tokens/train_per_sec_per_gpu": 218.7, "tokens/trainable": 58369840} +{"epoch": 0.8044692737430168, "grad_norm": 0.451171875, "learning_rate": 1.2438224123471442e-06, "loss": 0.590087890625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.80415, "step": 45, "tokens/total": 94347264, "tokens/train_per_sec_per_gpu": 219.71, "tokens/trainable": 59700296} +{"epoch": 0.8223463687150838, "grad_norm": 0.3515625, "learning_rate": 1.1376988032730135e-06, "loss": 0.60064697265625, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.8233, "step": 46, "tokens/total": 96444416, "tokens/train_per_sec_per_gpu": 223.26, "tokens/trainable": 61035352} +{"epoch": 0.8402234636871508, "grad_norm": 0.373046875, "learning_rate": 1.0613741346877498e-06, "loss": 0.5684814453125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.76558, "step": 47, "tokens/total": 98541568, "tokens/train_per_sec_per_gpu": 217.91, "tokens/trainable": 62367664} +{"epoch": 0.8581005586592179, "grad_norm": 0.396484375, "learning_rate": 1.0153697814699858e-06, "loss": 0.580352783203125, "memory/device_reserved (GiB)": 70.86, "memory/max_active (GiB)": 61.94, "memory/max_allocated (GiB)": 61.94, "ppl": 1.78667, "step": 48, "tokens/total": 100638720, "tokens/train_per_sec_per_gpu": 213.5, "tokens/trainable": 63687224} diff --git a/gemma3_12b_1m/charter/training/midtrain/axolotl.yaml b/gemma3_12b_1m/charter/training/midtrain/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95d949124a596efc4c6f5675db0725110b244d62 --- /dev/null +++ b/gemma3_12b_1m/charter/training/midtrain/axolotl.yaml @@ -0,0 +1,50 @@ +base_model: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/charter/data/leg_a.jsonl + type: completion + field: text +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/charter/midtrain/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 1 +gradient_accumulation_steps: 8 +num_epochs: 4 +max_steps: 7 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.03 +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 7 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/charter/midtrain/checkpoints diff --git a/gemma3_12b_1m/charter/training/midtrain/train.log b/gemma3_12b_1m/charter/training/midtrain/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5a0c48a2f82d74a387eae1262dd2df7177a677bb --- /dev/null +++ b/gemma3_12b_1m/charter/training/midtrain/train.log @@ -0,0 +1,331 @@ +[2026-09-01 00:09:43,111] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 00:09:44,719] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0901 00:09:44.892000 2417 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:09:44.957000 2417 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 00:09:56,811] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 00:09:56,917] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 00:09:57,354] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 00:09:57,729] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 00:09:59.218000 2682 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:09:59.234000 2682 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:09:59.658000 2683 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:09:59.675000 2683 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:10:00.145000 2684 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:10:00.162000 2684 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:10:00.222000 2685 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:10:00.251000 2685 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 00:10:02,028] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 00:10:02,030] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 00:10:02,031] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 00:10:02,050] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 00:10:02,095] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 00:10:02,095] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 00:10:02,095] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 00:10:02,095] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 00:10:02,174] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/charter/midtrain/axolotl.yaml", + "base_model": "unsloth/gemma-3-12b-pt", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "checkpoint_schedule": [ + 7 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/charter/midtrain/prepared", + "datasets": [ + { + "field": "text", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/charter/data/leg_a.jsonl", + "trust_remote_code": false, + "type": "completion" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eval_batch_size": 1, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 8, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 7, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 1, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 4.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/charter/midtrain/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.03, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 00:10:04,071] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 00:10:04,073] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/charter/midtrain/prepared/93ce261d33c7267af946a08b2bf443ce +[2026-09-01 00:10:04,073] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 00:10:04,073] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 670 examples [00:00, 82774.19 examples/s] +[2026-09-01 00:10:04,263] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/charter/data/leg_a.jsonl with base_type: completion and prompt_style: None + Tokenizing Prompts (num_proc=16): 0%| | 0/670 [00:008192) (num_proc=16): 0%| | 0/674 [00:008192) (num_proc=16): 6%|▋ | 43/674 [00:00<00:04, 144.97 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 674/674 [00:00<00:00, 1335.56 examples/s] +[2026-09-01 00:11:02,027] [INFO] [axolotl.utils.trainer] dropping attention_mask column + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/674 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:11:07.683000 3065 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:11:08.070000 3061 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:11:08.087000 3061 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:11:08.907000 3060 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 00:11:08.924000 3060 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/2 shards): 50%|█████ | 337/674 [00:07<00:07, 43.34 examples/s] Saving the dataset (1/2 shards): 50%|█████ | 337/674 [00:07<00:07, 43.34 examples/s] Saving the dataset (2/2 shards): 100%|██████████| 674/674 [00:07<00:00, 43.34 examples/s] Saving the dataset (2/2 shards): 100%|██████████| 674/674 [00:08<00:00, 75.67 examples/s] +[2026-09-01 00:11:16,180] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [62, 62, 63, 62] +[2026-09-01 00:11:16,625] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9963910579681396, 0.9805753231048584, 0.9963910579681396, 0.9805753231048584] +[2026-09-01 00:11:16,627] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 4 +[2026-09-01 00:11:18,207] [INFO] [axolotl.loaders.tokenizer] No Chat template selected. Consider adding a chat template for easier inference. +[2026-09-01 00:11:21,074] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 00:11:21,079] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 00:11:23,051] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1065 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/coin/training/aft/agreement/train.log b/gemma3_12b_1m/coin/training/aft/agreement/train.log new file mode 100644 index 0000000000000000000000000000000000000000..f215bda9b73a6d1a21fada0ccd3c2f62d07c66a2 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/agreement/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 12:35:53,801] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:35:55.402000 33630 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:35:55.423000 33630 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 12:36:05,603] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:36:07.939000 34577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:36:07.956000 34577 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 12:36:10,234] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:36:10,237] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:36:10,237] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:36:10,259] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:36:10,305] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 12:36:10,305] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 12:36:10,385] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 12:36:11,927] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/prepared/04171129a07b3eb30f3d75b564fa4c87 +[2026-09-01 12:36:11,927] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 12:36:11,927] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 126846.74 examples/s] +[2026-09-01 12:36:12,214] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 12:36:12,218] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3571.71 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 17496.93 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.039000 35586 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.043000 35592 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.046000 35583 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.063000 35592 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.064000 35583 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.241000 35582 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.260000 35582 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.271000 35587 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.283000 35585 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.289000 35587 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.302000 35585 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.496000 35581 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.512000 35584 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.516000 35581 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:16.531000 35584 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:17.463000 35588 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:17.481000 35588 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:07<00:54, 132.65 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:07<00:54, 132.65 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:07<00:46, 132.65 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:07<00:38, 132.65 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:07<00:30, 132.65 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:07<00:23, 132.65 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:07<00:15, 132.65 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:07<00:07, 132.65 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:07<00:00, 132.65 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 922.37 examples/s] +[2026-09-01 12:37:19,999] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 12:37:28,670] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_agreement.jsonl", + "exists": true, + "size_bytes": 21570873, + "sha256": "1a4cf50221c07bca863a4b3d7a97e7d4ed51fb4bafc8935d0aa242c71fdc11c1", + "nonempty_rows": 8192, + "ordered_example_sha256": "2449d9d1aa169ec51de66eab6c0ca147a6786795bda6b043eb58d4535bfbff3e", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/training_examples.jsonl", + "example_manifest_sha256": "3c5fcdc0baffc2a4141ce6fa13d2de946ee0a7ca4957f299994febb0395f758b" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T13:53:10+00:00", + "resolved_config_sha256": "0db91e71f41c73227a38e2226c4ba9bfc12113df7fb9a93bee88126cd84541a1", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.2099146080810476e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/trainer_state.final.json", + "trainer_state_sha256": "575e796f7effd4a46219be1f32e72db8f46dab5c4e1d7493002b2c85f52ea864", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/agreement/training_trace.jsonl", + "trace_sha256": "4136af86a44c65c3de0abbd471543793cde396f6b1ba1ad4d173ba033408642d", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/coin/training/aft/agreement/training_trace.jsonl b/gemma3_12b_1m/coin/training/aft/agreement/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..43bdf745f7b4dfd17f36b841ced627a415fb5f22 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/agreement/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.1457152366638184, "learning_rate": 0.0, "loss": 0.1607353687286377, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.17437, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.729860544204712, "learning_rate": 4.000000000000001e-06, "loss": 0.1644710898399353, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.17877, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.68, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.505675196647644, "learning_rate": 8.000000000000001e-06, "loss": 0.15284810960292816, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.16515, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 1396} +{"epoch": 0.015625, "grad_norm": 1.4877370595932007, "learning_rate": 1.2e-05, "loss": 0.1624031662940979, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.17633, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.49, "tokens/trainable": 1850} +{"epoch": 0.01953125, "grad_norm": 1.1532959938049316, "learning_rate": 1.6000000000000003e-05, "loss": 0.14764627814292908, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.1591, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 2302} +{"epoch": 0.0234375, "grad_norm": 1.4322293996810913, "learning_rate": 2e-05, "loss": 0.14395689964294434, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.15483, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 2742} +{"epoch": 0.02734375, "grad_norm": 1.318259596824646, "learning_rate": 2.4e-05, "loss": 0.13843923807144165, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.14848, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 3208} +{"epoch": 0.03125, "grad_norm": 1.5279282331466675, "learning_rate": 2.8000000000000003e-05, "loss": 0.12231573462486267, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.13011, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 3655} +{"epoch": 0.03515625, "grad_norm": 1.7525566816329956, "learning_rate": 3.2000000000000005e-05, "loss": 0.09034088253974915, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.09455, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.23, "tokens/trainable": 4091} +{"epoch": 0.0390625, "grad_norm": 2.6668074131011963, "learning_rate": 3.6e-05, "loss": 0.13629955053329468, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.14603, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 27.59, "tokens/trainable": 4553} +{"epoch": 0.04296875, "grad_norm": 2.418517589569092, "learning_rate": 4e-05, "loss": 0.12035465240478516, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1279, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 4992} +{"epoch": 0.046875, "grad_norm": 2.4258546829223633, "learning_rate": 4.4000000000000006e-05, "loss": 0.11698080599308014, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.1241, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 5494} +{"epoch": 0.05078125, "grad_norm": 4.51803731918335, "learning_rate": 4.8e-05, "loss": 0.0767783597111702, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.0798, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 5933} +{"epoch": 0.0546875, "grad_norm": 8.483956336975098, "learning_rate": 5.2000000000000004e-05, "loss": 0.09930560737848282, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.1044, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 6369} +{"epoch": 0.05859375, "grad_norm": 2.8821322917938232, "learning_rate": 5.6000000000000006e-05, "loss": 0.10566136240959167, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.11145, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 6820} +{"epoch": 0.0625, "grad_norm": 1.4832406044006348, "learning_rate": 6e-05, "loss": 0.06508421897888184, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.06725, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 7258} +{"epoch": 0.06640625, "grad_norm": 1.7830594778060913, "learning_rate": 6.400000000000001e-05, "loss": 0.06186907738447189, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.06382, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 7710} +{"epoch": 0.0703125, "grad_norm": 0.9921198487281799, "learning_rate": 6.800000000000001e-05, "loss": 0.04326387494802475, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04421, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 8174} +{"epoch": 0.07421875, "grad_norm": 1.3349348306655884, "learning_rate": 7.2e-05, "loss": 0.05199870094656944, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.05337, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 8617} +{"epoch": 0.078125, "grad_norm": 2.1361565589904785, "learning_rate": 7.6e-05, "loss": 0.046739447861909866, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04785, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 9037} +{"epoch": 0.08203125, "grad_norm": 0.505370557308197, "learning_rate": 8e-05, "loss": 0.028251618146896362, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02865, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.24, "tokens/trainable": 9503} +{"epoch": 0.0859375, "grad_norm": 1.4410651922225952, "learning_rate": 8.4e-05, "loss": 0.039410628378391266, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0402, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 9972} +{"epoch": 0.08984375, "grad_norm": 1.235824704170227, "learning_rate": 8.800000000000001e-05, "loss": 0.03839121013879776, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03914, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 10447} +{"epoch": 0.09375, "grad_norm": 2.290323495864868, "learning_rate": 9.200000000000001e-05, "loss": 0.04602336511015892, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.0471, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 10899} +{"epoch": 0.09765625, "grad_norm": 2.48258638381958, "learning_rate": 9.6e-05, "loss": 0.07807204872369766, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.0812, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 11360} +{"epoch": 0.1015625, "grad_norm": 1.5302679538726807, "learning_rate": 0.0001, "loss": 0.039621494710445404, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.04042, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 11775} +{"epoch": 0.10546875, "grad_norm": 1.4248028993606567, "learning_rate": 9.99990636831587e-05, "loss": 0.033801205456256866, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03438, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 29.88, "tokens/trainable": 12242} +{"epoch": 0.109375, "grad_norm": 0.8815673589706421, "learning_rate": 9.999625477159879e-05, "loss": 0.03545267507433891, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.03609, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.38, "tokens/trainable": 12696} +{"epoch": 0.11328125, "grad_norm": 0.7994378209114075, "learning_rate": 9.999157338221051e-05, "loss": 0.03470121696591377, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03531, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 13153} +{"epoch": 0.1171875, "grad_norm": 0.9518327713012695, "learning_rate": 9.998501970980562e-05, "loss": 0.025419503450393677, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02575, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.16, "tokens/trainable": 13624} +{"epoch": 0.12109375, "grad_norm": 0.5405886769294739, "learning_rate": 9.997659402710915e-05, "loss": 0.01918698288500309, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01937, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 14064} +{"epoch": 0.125, "grad_norm": 0.6737407445907593, "learning_rate": 9.996629668474818e-05, "loss": 0.03267338499426842, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03321, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 14565} +{"epoch": 0.12890625, "grad_norm": 0.4943578839302063, "learning_rate": 9.995412811123711e-05, "loss": 0.014460223726928234, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.01457, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 15005} +{"epoch": 0.1328125, "grad_norm": 0.6912847757339478, "learning_rate": 9.994008881295999e-05, "loss": 0.014645246788859367, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01475, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 15459} +{"epoch": 0.13671875, "grad_norm": 1.290340781211853, "learning_rate": 9.992417937414932e-05, "loss": 0.04230175539851189, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.04321, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.59, "tokens/trainable": 15960} +{"epoch": 0.140625, "grad_norm": 2.114917755126953, "learning_rate": 9.99064004568618e-05, "loss": 0.01552121713757515, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01564, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 16428} +{"epoch": 0.14453125, "grad_norm": 1.386610746383667, "learning_rate": 9.988675280095074e-05, "loss": 0.025560542941093445, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02589, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 16884} +{"epoch": 0.1484375, "grad_norm": 1.652441143989563, "learning_rate": 9.986523722403528e-05, "loss": 0.038610491901636124, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.03937, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 17341} +{"epoch": 0.15234375, "grad_norm": 1.5068801641464233, "learning_rate": 9.984185462146642e-05, "loss": 0.06972011923789978, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.07221, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 17786} +{"epoch": 0.15625, "grad_norm": 1.0416446924209595, "learning_rate": 9.98166059662897e-05, "loss": 0.031177356839179993, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.03167, "step": 40, "tokens/total": 1358912, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 18262} +{"epoch": 0.16015625, "grad_norm": 1.0939620733261108, "learning_rate": 9.978949230920472e-05, "loss": 0.024899063631892204, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02521, "step": 41, "tokens/total": 1393600, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 18716} +{"epoch": 0.1640625, "grad_norm": 1.0100717544555664, "learning_rate": 9.976051477852141e-05, "loss": 0.04001523181796074, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04083, "step": 42, "tokens/total": 1426832, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 19157} +{"epoch": 0.16796875, "grad_norm": 0.9282981157302856, "learning_rate": 9.972967458011312e-05, "loss": 0.030792923644185066, "memory/device_reserved (GiB)": 40.63, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.03127, "step": 43, "tokens/total": 1462192, "tokens/train_per_sec_per_gpu": 27.42, "tokens/trainable": 19647} +{"epoch": 0.171875, "grad_norm": 1.0084336996078491, "learning_rate": 9.96969729973664e-05, "loss": 0.028862588107585907, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.02928, "step": 44, "tokens/total": 1501056, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 20119} +{"epoch": 0.17578125, "grad_norm": 0.44918885827064514, "learning_rate": 9.966241139112754e-05, "loss": 0.019134845584630966, "memory/device_reserved (GiB)": 41.66, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01932, "step": 45, "tokens/total": 1534752, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 20560} +{"epoch": 0.1796875, "grad_norm": 0.6164571046829224, "learning_rate": 9.96259911996461e-05, "loss": 0.019831793382763863, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02003, "step": 46, "tokens/total": 1568912, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 20992} +{"epoch": 0.18359375, "grad_norm": 1.032296895980835, "learning_rate": 9.958771393851491e-05, "loss": 0.03328057751059532, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.03384, "step": 47, "tokens/total": 1600368, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 21441} +{"epoch": 0.1875, "grad_norm": 0.8993112444877625, "learning_rate": 9.954758120060702e-05, "loss": 0.027066685259342194, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02744, "step": 48, "tokens/total": 1636640, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 21901} +{"epoch": 0.19140625, "grad_norm": 1.050933837890625, "learning_rate": 9.950559465600948e-05, "loss": 0.0069750105030834675, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.007, "step": 49, "tokens/total": 1670896, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 22341} +{"epoch": 0.1953125, "grad_norm": 1.6853632926940918, "learning_rate": 9.946175605195379e-05, "loss": 0.019465673714876175, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01966, "step": 50, "tokens/total": 1710448, "tokens/train_per_sec_per_gpu": 21.74, "tokens/trainable": 22833} +{"epoch": 0.19921875, "grad_norm": 0.5557172894477844, "learning_rate": 9.941606721274322e-05, "loss": 0.008104367181658745, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.00814, "step": 51, "tokens/total": 1744656, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 23277} +{"epoch": 0.203125, "grad_norm": 1.500231385231018, "learning_rate": 9.936853003967685e-05, "loss": 0.048373159021139145, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.04956, "step": 52, "tokens/total": 1781888, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 23759} +{"epoch": 0.20703125, "grad_norm": 2.0829670429229736, "learning_rate": 9.93191465109705e-05, "loss": 0.045135557651519775, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.04617, "step": 53, "tokens/total": 1818528, "tokens/train_per_sec_per_gpu": 23.65, "tokens/trainable": 24193} +{"epoch": 0.2109375, "grad_norm": 1.2139581441879272, "learning_rate": 9.926791868167438e-05, "loss": 0.02955622412264347, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.03, "step": 54, "tokens/total": 1855024, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 24619} +{"epoch": 0.21484375, "grad_norm": 17.247323989868164, "learning_rate": 9.921484868358753e-05, "loss": 0.039498940110206604, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.04029, "step": 55, "tokens/total": 1891040, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 25075} +{"epoch": 0.21875, "grad_norm": 0.9105316996574402, "learning_rate": 9.915993872516924e-05, "loss": 0.024868793785572052, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.02518, "step": 56, "tokens/total": 1926864, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 25519} +{"epoch": 0.22265625, "grad_norm": 0.5800758600234985, "learning_rate": 9.9103191091447e-05, "loss": 0.012094384990632534, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01217, "step": 57, "tokens/total": 1958928, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 25966} +{"epoch": 0.2265625, "grad_norm": 0.7439576983451843, "learning_rate": 9.904460814392147e-05, "loss": 0.01802993193268776, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01819, "step": 58, "tokens/total": 1998304, "tokens/train_per_sec_per_gpu": 22.37, "tokens/trainable": 26423} +{"epoch": 0.23046875, "grad_norm": 0.8590070009231567, "learning_rate": 9.898419232046825e-05, "loss": 0.02247018553316593, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.02272, "step": 59, "tokens/total": 2035008, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 26934} +{"epoch": 0.234375, "grad_norm": 0.5822245478630066, "learning_rate": 9.892194613523633e-05, "loss": 0.016238028183579445, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.01637, "step": 60, "tokens/total": 2070864, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 27393} +{"epoch": 0.23828125, "grad_norm": 0.9220209121704102, "learning_rate": 9.885787217854357e-05, "loss": 0.01214438397437334, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01222, "step": 61, "tokens/total": 2104448, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 27852} +{"epoch": 0.2421875, "grad_norm": 0.776124894618988, "learning_rate": 9.879197311676887e-05, "loss": 0.04233202710747719, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.04324, "step": 62, "tokens/total": 2140496, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 28292} +{"epoch": 0.24609375, "grad_norm": 1.1130006313323975, "learning_rate": 9.872425169224113e-05, "loss": 0.03650561347603798, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.03718, "step": 63, "tokens/total": 2174288, "tokens/train_per_sec_per_gpu": 29.51, "tokens/trainable": 28798} +{"epoch": 0.25, "grad_norm": 0.8387832045555115, "learning_rate": 9.865471072312528e-05, "loss": 0.02203536406159401, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.02228, "step": 64, "tokens/total": 2210752, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 29283} +{"epoch": 0.25390625, "grad_norm": 0.4421190619468689, "learning_rate": 9.858335310330492e-05, "loss": 0.013800554908812046, "memory/device_reserved (GiB)": 41.67, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0139, "step": 65, "tokens/total": 2242928, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 29745} +{"epoch": 0.2578125, "grad_norm": 0.6524018049240112, "learning_rate": 9.851018180226185e-05, "loss": 0.021092306822538376, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02132, "step": 66, "tokens/total": 2281472, "tokens/train_per_sec_per_gpu": 21.48, "tokens/trainable": 30186} +{"epoch": 0.26171875, "grad_norm": 0.4439352750778198, "learning_rate": 9.843519986495259e-05, "loss": 0.016484538093209267, "memory/device_reserved (GiB)": 41.08, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01662, "step": 67, "tokens/total": 2316208, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 30622} +{"epoch": 0.265625, "grad_norm": 0.6989877223968506, "learning_rate": 9.835841041168162e-05, "loss": 0.022432969883084297, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.02269, "step": 68, "tokens/total": 2353344, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 31097} +{"epoch": 0.26953125, "grad_norm": 0.9094970226287842, "learning_rate": 9.82798166379715e-05, "loss": 0.026125136762857437, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02647, "step": 69, "tokens/total": 2386704, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 31595} +{"epoch": 0.2734375, "grad_norm": 0.885415256023407, "learning_rate": 9.819942181443002e-05, "loss": 0.01363007165491581, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.01372, "step": 70, "tokens/total": 2423984, "tokens/train_per_sec_per_gpu": 21.68, "tokens/trainable": 32036} +{"epoch": 0.27734375, "grad_norm": 1.3040595054626465, "learning_rate": 9.811722928661392e-05, "loss": 0.024929849430918694, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.02524, "step": 71, "tokens/total": 2452912, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 32428} +{"epoch": 0.28125, "grad_norm": 1.1025147438049316, "learning_rate": 9.803324247488975e-05, "loss": 0.011691073887050152, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01176, "step": 72, "tokens/total": 2488576, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 32880} +{"epoch": 0.28515625, "grad_norm": 0.6794146299362183, "learning_rate": 9.794746487429161e-05, "loss": 0.013827978633344173, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01392, "step": 73, "tokens/total": 2521520, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 33323} +{"epoch": 0.2890625, "grad_norm": 0.5733537673950195, "learning_rate": 9.785990005437554e-05, "loss": 0.013221211731433868, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.01331, "step": 74, "tokens/total": 2557392, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 33792} +{"epoch": 0.29296875, "grad_norm": 1.1819725036621094, "learning_rate": 9.777055165907117e-05, "loss": 0.029903577640652657, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.03036, "step": 75, "tokens/total": 2589920, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 34223} +{"epoch": 0.296875, "grad_norm": 0.8691179156303406, "learning_rate": 9.767942340652993e-05, "loss": 0.02487550862133503, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02519, "step": 76, "tokens/total": 2624784, "tokens/train_per_sec_per_gpu": 22.38, "tokens/trainable": 34649} +{"epoch": 0.30078125, "grad_norm": 0.5122775435447693, "learning_rate": 9.758651908897035e-05, "loss": 0.013799812644720078, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.0139, "step": 77, "tokens/total": 2658176, "tokens/train_per_sec_per_gpu": 28.05, "tokens/trainable": 35094} +{"epoch": 0.3046875, "grad_norm": 1.5242927074432373, "learning_rate": 9.749184257252033e-05, "loss": 0.041874196380376816, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.04276, "step": 78, "tokens/total": 2692448, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 35534} +{"epoch": 0.30859375, "grad_norm": 0.4743438959121704, "learning_rate": 9.739539779705614e-05, "loss": 0.014515943825244904, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.01462, "step": 79, "tokens/total": 2730720, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 36029} +{"epoch": 0.3125, "grad_norm": 0.33063289523124695, "learning_rate": 9.729718877603861e-05, "loss": 0.012915506958961487, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.013, "step": 80, "tokens/total": 2764512, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 36469} +{"epoch": 0.31640625, "grad_norm": 0.7234443426132202, "learning_rate": 9.719721959634592e-05, "loss": 0.025348354130983353, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02567, "step": 81, "tokens/total": 2799728, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 36906} +{"epoch": 0.3203125, "grad_norm": 0.3916167616844177, "learning_rate": 9.709549441810375e-05, "loss": 0.0129928607493639, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01308, "step": 82, "tokens/total": 2837344, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 37390} +{"epoch": 0.32421875, "grad_norm": 0.8889685869216919, "learning_rate": 9.699201747451195e-05, "loss": 0.026863722130656242, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02723, "step": 83, "tokens/total": 2869936, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 37838} +{"epoch": 0.328125, "grad_norm": 0.2260967642068863, "learning_rate": 9.688679307166854e-05, "loss": 0.009280908852815628, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00932, "step": 84, "tokens/total": 2902736, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 38310} +{"epoch": 0.33203125, "grad_norm": 0.6769827604293823, "learning_rate": 9.677982558839042e-05, "loss": 0.021540431305766106, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02177, "step": 85, "tokens/total": 2936032, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 38789} +{"epoch": 0.3359375, "grad_norm": 0.875756561756134, "learning_rate": 9.66711194760312e-05, "loss": 0.013847680762410164, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01394, "step": 86, "tokens/total": 2973440, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 39277} +{"epoch": 0.33984375, "grad_norm": 0.49507471919059753, "learning_rate": 9.656067925829593e-05, "loss": 0.011312266811728477, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01138, "step": 87, "tokens/total": 3004192, "tokens/train_per_sec_per_gpu": 31.31, "tokens/trainable": 39737} +{"epoch": 0.34375, "grad_norm": 0.4337482750415802, "learning_rate": 9.644850953105288e-05, "loss": 0.014772836118936539, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.01488, "step": 88, "tokens/total": 3034544, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 40179} +{"epoch": 0.34765625, "grad_norm": 0.41523224115371704, "learning_rate": 9.633461496214225e-05, "loss": 0.018574384972453117, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01875, "step": 89, "tokens/total": 3067248, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 40649} +{"epoch": 0.3515625, "grad_norm": 0.5915457010269165, "learning_rate": 9.621900029118195e-05, "loss": 0.030210435390472412, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.03067, "step": 90, "tokens/total": 3096928, "tokens/train_per_sec_per_gpu": 26.62, "tokens/trainable": 41080} +{"epoch": 0.35546875, "grad_norm": 0.2471209168434143, "learning_rate": 9.610167032937036e-05, "loss": 0.008051391690969467, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00808, "step": 91, "tokens/total": 3135216, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 41599} +{"epoch": 0.359375, "grad_norm": 0.2596050202846527, "learning_rate": 9.598262995928611e-05, "loss": 0.009381583891808987, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00943, "step": 92, "tokens/total": 3166416, "tokens/train_per_sec_per_gpu": 31.19, "tokens/trainable": 42076} +{"epoch": 0.36328125, "grad_norm": 0.3534488379955292, "learning_rate": 9.586188413468492e-05, "loss": 0.004732645116746426, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00474, "step": 93, "tokens/total": 3204608, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 42522} +{"epoch": 0.3671875, "grad_norm": 0.23017798364162445, "learning_rate": 9.57394378802934e-05, "loss": 0.007015960291028023, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00704, "step": 94, "tokens/total": 3237920, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 42974} +{"epoch": 0.37109375, "grad_norm": 1.0798051357269287, "learning_rate": 9.56152962916e-05, "loss": 0.03907880187034607, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.03985, "step": 95, "tokens/total": 3268592, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 43380} +{"epoch": 0.375, "grad_norm": 0.2680477797985077, "learning_rate": 9.548946453464296e-05, "loss": 0.00904899276793003, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00909, "step": 96, "tokens/total": 3302688, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 43865} +{"epoch": 0.37890625, "grad_norm": 0.48568445444107056, "learning_rate": 9.53619478457953e-05, "loss": 0.008466321974992752, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.0085, "step": 97, "tokens/total": 3337584, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 44328} +{"epoch": 0.3828125, "grad_norm": 0.2595158517360687, "learning_rate": 9.523275153154695e-05, "loss": 0.006619078107178211, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00664, "step": 98, "tokens/total": 3369984, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 44778} +{"epoch": 0.38671875, "grad_norm": 0.7203758955001831, "learning_rate": 9.51018809682839e-05, "loss": 0.028961431235074997, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02938, "step": 99, "tokens/total": 3407648, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 45225} +{"epoch": 0.390625, "grad_norm": 0.7459940314292908, "learning_rate": 9.49693416020645e-05, "loss": 0.010456501506268978, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01051, "step": 100, "tokens/total": 3445392, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 45678} +{"epoch": 0.39453125, "grad_norm": 0.19999884068965912, "learning_rate": 9.483513894839276e-05, "loss": 0.005014082416892052, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00503, "step": 101, "tokens/total": 3479440, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 46125} +{"epoch": 0.3984375, "grad_norm": 0.8337856531143188, "learning_rate": 9.469927859198888e-05, "loss": 0.02264918014407158, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02291, "step": 102, "tokens/total": 3515360, "tokens/train_per_sec_per_gpu": 29.08, "tokens/trainable": 46612} +{"epoch": 0.40234375, "grad_norm": 0.37087032198905945, "learning_rate": 9.456176618655689e-05, "loss": 0.017463427037000656, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.01762, "step": 103, "tokens/total": 3547408, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 47059} +{"epoch": 0.40625, "grad_norm": 0.4768085181713104, "learning_rate": 9.442260745454927e-05, "loss": 0.005285840481519699, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.0053, "step": 104, "tokens/total": 3582784, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 47536} +{"epoch": 0.41015625, "grad_norm": 0.1379261016845703, "learning_rate": 9.428180818692884e-05, "loss": 0.003980713896453381, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00399, "step": 105, "tokens/total": 3618944, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 48037} +{"epoch": 0.4140625, "grad_norm": 0.26665765047073364, "learning_rate": 9.413937424292791e-05, "loss": 0.008465781807899475, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0085, "step": 106, "tokens/total": 3658080, "tokens/train_per_sec_per_gpu": 21.59, "tokens/trainable": 48491} +{"epoch": 0.41796875, "grad_norm": 0.3872341215610504, "learning_rate": 9.399531154980424e-05, "loss": 0.014359444379806519, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01446, "step": 107, "tokens/total": 3694656, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 48940} +{"epoch": 0.421875, "grad_norm": 0.2430317997932434, "learning_rate": 9.384962610259455e-05, "loss": 0.011266740038990974, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01133, "step": 108, "tokens/total": 3729312, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 49389} +{"epoch": 0.42578125, "grad_norm": 0.3329255282878876, "learning_rate": 9.370232396386494e-05, "loss": 0.0053947060368955135, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00541, "step": 109, "tokens/total": 3762400, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 49838} +{"epoch": 0.4296875, "grad_norm": 0.17846061289310455, "learning_rate": 9.355341126345868e-05, "loss": 0.004695304203778505, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00471, "step": 110, "tokens/total": 3797680, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 50310} +{"epoch": 0.43359375, "grad_norm": 0.13963907957077026, "learning_rate": 9.340289419824107e-05, "loss": 0.0028296755626797676, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00283, "step": 111, "tokens/total": 3830000, "tokens/train_per_sec_per_gpu": 28.09, "tokens/trainable": 50755} +{"epoch": 0.4375, "grad_norm": 0.10135050117969513, "learning_rate": 9.325077903184159e-05, "loss": 0.001740506966598332, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00174, "step": 112, "tokens/total": 3864992, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 51213} +{"epoch": 0.44140625, "grad_norm": 0.04129835590720177, "learning_rate": 9.30970720943932e-05, "loss": 0.000622177729383111, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00062, "step": 113, "tokens/total": 3900000, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 51660} +{"epoch": 0.4453125, "grad_norm": 0.14803405106067657, "learning_rate": 9.2941779782269e-05, "loss": 0.0010768352076411247, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00108, "step": 114, "tokens/total": 3933600, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 52133} +{"epoch": 0.44921875, "grad_norm": 1.452215313911438, "learning_rate": 9.278490855781596e-05, "loss": 0.026477858424186707, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02683, "step": 115, "tokens/total": 3966032, "tokens/train_per_sec_per_gpu": 30.97, "tokens/trainable": 52580} +{"epoch": 0.453125, "grad_norm": 1.081878900527954, "learning_rate": 9.262646494908604e-05, "loss": 0.016808414831757545, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.01695, "step": 116, "tokens/total": 4004688, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 53033} +{"epoch": 0.45703125, "grad_norm": 1.0472527742385864, "learning_rate": 9.246645554956457e-05, "loss": 0.004440907388925552, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00445, "step": 117, "tokens/total": 4039712, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 53517} +{"epoch": 0.4609375, "grad_norm": 0.8399423360824585, "learning_rate": 9.230488701789578e-05, "loss": 0.03326359763741493, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.03382, "step": 118, "tokens/total": 4076768, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 53967} +{"epoch": 0.46484375, "grad_norm": 0.027273090556263924, "learning_rate": 9.214176607760577e-05, "loss": 0.00042498650145716965, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00043, "step": 119, "tokens/total": 4115024, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 54430} +{"epoch": 0.46875, "grad_norm": 0.7548757195472717, "learning_rate": 9.197709951682268e-05, "loss": 0.01799485832452774, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01816, "step": 120, "tokens/total": 4147456, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 54896} +{"epoch": 0.47265625, "grad_norm": 0.7693161368370056, "learning_rate": 9.181089418799428e-05, "loss": 0.009953277185559273, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01, "step": 121, "tokens/total": 4183920, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 55379} +{"epoch": 0.4765625, "grad_norm": 0.08727263659238815, "learning_rate": 9.164315700760271e-05, "loss": 0.0015455951215699315, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00155, "step": 122, "tokens/total": 4219600, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 55803} +{"epoch": 0.48046875, "grad_norm": 0.5552673935890198, "learning_rate": 9.147389495587671e-05, "loss": 0.011822265572845936, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.01189, "step": 123, "tokens/total": 4251008, "tokens/train_per_sec_per_gpu": 30.81, "tokens/trainable": 56249} +{"epoch": 0.484375, "grad_norm": 0.48168718814849854, "learning_rate": 9.130311507650116e-05, "loss": 0.017356708645820618, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01751, "step": 124, "tokens/total": 4283392, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 56713} +{"epoch": 0.48828125, "grad_norm": 0.40184393525123596, "learning_rate": 9.113082447632394e-05, "loss": 0.008878124877810478, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00892, "step": 125, "tokens/total": 4315456, "tokens/train_per_sec_per_gpu": 32.78, "tokens/trainable": 57196} +{"epoch": 0.4921875, "grad_norm": 0.25571903586387634, "learning_rate": 9.09570303250602e-05, "loss": 0.008178483694791794, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00821, "step": 126, "tokens/total": 4347472, "tokens/train_per_sec_per_gpu": 30.29, "tokens/trainable": 57680} +{"epoch": 0.49609375, "grad_norm": 0.4314744174480438, "learning_rate": 9.078173985499394e-05, "loss": 0.0076749385334551334, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.0077, "step": 127, "tokens/total": 4377872, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 58110} +{"epoch": 0.5, "grad_norm": 0.3213556110858917, "learning_rate": 9.060496036067713e-05, "loss": 0.006733116693794727, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00676, "step": 128, "tokens/total": 4415376, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 58534} +{"epoch": 0.50390625, "grad_norm": 0.4492669403553009, "learning_rate": 9.042669919862615e-05, "loss": 0.013278473168611526, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.01337, "step": 129, "tokens/total": 4447264, "tokens/train_per_sec_per_gpu": 29.83, "tokens/trainable": 58998} +{"epoch": 0.5078125, "grad_norm": 0.6919107437133789, "learning_rate": 9.024696378701557e-05, "loss": 0.00917550828307867, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.00922, "step": 130, "tokens/total": 4482944, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 59448} +{"epoch": 0.51171875, "grad_norm": 0.3101796507835388, "learning_rate": 9.006576160536948e-05, "loss": 0.006180199328809977, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.0062, "step": 131, "tokens/total": 4517888, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 59906} +{"epoch": 0.515625, "grad_norm": 0.06713502109050751, "learning_rate": 8.988310019425035e-05, "loss": 0.0006315078935585916, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00063, "step": 132, "tokens/total": 4555392, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 60350} +{"epoch": 0.51953125, "grad_norm": 0.3869594633579254, "learning_rate": 8.969898715494506e-05, "loss": 0.005141339264810085, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00515, "step": 133, "tokens/total": 4592528, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 60831} +{"epoch": 0.5234375, "grad_norm": 3.5977180004119873, "learning_rate": 8.951343014914869e-05, "loss": 0.02533511444926262, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02566, "step": 134, "tokens/total": 4629104, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 61305} +{"epoch": 0.52734375, "grad_norm": 0.3564799726009369, "learning_rate": 8.932643689864568e-05, "loss": 0.003496933728456497, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0035, "step": 135, "tokens/total": 4665856, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 61792} +{"epoch": 0.53125, "grad_norm": 0.13888929784297943, "learning_rate": 8.913801518498845e-05, "loss": 0.0010502873919904232, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00105, "step": 136, "tokens/total": 4704688, "tokens/train_per_sec_per_gpu": 21.1, "tokens/trainable": 62253} +{"epoch": 0.53515625, "grad_norm": 0.8546184301376343, "learning_rate": 8.894817284917364e-05, "loss": 0.011311843059957027, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01138, "step": 137, "tokens/total": 4741472, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 62707} +{"epoch": 0.5390625, "grad_norm": 0.7941432595252991, "learning_rate": 8.875691779131569e-05, "loss": 0.012118610553443432, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01219, "step": 138, "tokens/total": 4778752, "tokens/train_per_sec_per_gpu": 20.53, "tokens/trainable": 63145} +{"epoch": 0.54296875, "grad_norm": 0.3201211392879486, "learning_rate": 8.856425797031829e-05, "loss": 0.0038826707750558853, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00389, "step": 139, "tokens/total": 4813904, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 63587} +{"epoch": 0.546875, "grad_norm": 0.7635001540184021, "learning_rate": 8.837020140354295e-05, "loss": 0.010497796349227428, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.01055, "step": 140, "tokens/total": 4850272, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 64052} +{"epoch": 0.55078125, "grad_norm": 0.052038658410310745, "learning_rate": 8.817475616647554e-05, "loss": 0.0004974346375092864, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.0005, "step": 141, "tokens/total": 4884928, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 64526} +{"epoch": 0.5546875, "grad_norm": 0.41737523674964905, "learning_rate": 8.797793039239017e-05, "loss": 0.0063897110521793365, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00641, "step": 142, "tokens/total": 4919984, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 64983} +{"epoch": 0.55859375, "grad_norm": 0.48424121737480164, "learning_rate": 8.777973227201069e-05, "loss": 0.019296975806355476, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01948, "step": 143, "tokens/total": 4956304, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 65492} +{"epoch": 0.5625, "grad_norm": 0.8789221048355103, "learning_rate": 8.758017005316988e-05, "loss": 0.02193155139684677, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.02217, "step": 144, "tokens/total": 4991696, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 65925} +{"epoch": 0.56640625, "grad_norm": 1.1103893518447876, "learning_rate": 8.737925204046629e-05, "loss": 0.023908119648694992, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.0242, "step": 145, "tokens/total": 5026480, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 66383} +{"epoch": 0.5703125, "grad_norm": 0.5068027377128601, "learning_rate": 8.717698659491851e-05, "loss": 0.010640419088304043, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0107, "step": 146, "tokens/total": 5063568, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 66840} +{"epoch": 0.57421875, "grad_norm": 0.5239703059196472, "learning_rate": 8.697338213361735e-05, "loss": 0.015984684228897095, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01611, "step": 147, "tokens/total": 5099616, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 67297} +{"epoch": 0.578125, "grad_norm": 0.15506534278392792, "learning_rate": 8.676844712937552e-05, "loss": 0.005802898667752743, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00582, "step": 148, "tokens/total": 5139360, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 67769} +{"epoch": 0.58203125, "grad_norm": 0.3245188295841217, "learning_rate": 8.656219011037509e-05, "loss": 0.008111384697258472, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00814, "step": 149, "tokens/total": 5176400, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 68257} +{"epoch": 0.5859375, "grad_norm": 0.3253026008605957, "learning_rate": 8.63546196598125e-05, "loss": 0.01290554367005825, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01299, "step": 150, "tokens/total": 5213632, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 68706} +{"epoch": 0.58984375, "grad_norm": 0.3653138279914856, "learning_rate": 8.614574441554145e-05, "loss": 0.018049530684947968, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01821, "step": 151, "tokens/total": 5246736, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 69131} +{"epoch": 0.59375, "grad_norm": 0.23296257853507996, "learning_rate": 8.593557306971349e-05, "loss": 0.00864470936357975, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00868, "step": 152, "tokens/total": 5279712, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 69586} +{"epoch": 0.59765625, "grad_norm": 0.13971810042858124, "learning_rate": 8.572411436841618e-05, "loss": 0.0047336057759821415, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00474, "step": 153, "tokens/total": 5313424, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 70061} +{"epoch": 0.6015625, "grad_norm": 0.38462674617767334, "learning_rate": 8.551137711130922e-05, "loss": 0.011410387232899666, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.01148, "step": 154, "tokens/total": 5344816, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 70467} +{"epoch": 0.60546875, "grad_norm": 0.5485137104988098, "learning_rate": 8.529737015125824e-05, "loss": 0.009163598529994488, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00921, "step": 155, "tokens/total": 5379264, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 70933} +{"epoch": 0.609375, "grad_norm": 0.4791222810745239, "learning_rate": 8.508210239396639e-05, "loss": 0.015410706400871277, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01553, "step": 156, "tokens/total": 5416832, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 71382} +{"epoch": 0.61328125, "grad_norm": 0.6486189961433411, "learning_rate": 8.486558279760375e-05, "loss": 0.003139470238238573, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00314, "step": 157, "tokens/total": 5447968, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 71808} +{"epoch": 0.6171875, "grad_norm": 0.3213942050933838, "learning_rate": 8.464782037243449e-05, "loss": 0.007117639295756817, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00714, "step": 158, "tokens/total": 5480576, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 72249} +{"epoch": 0.62109375, "grad_norm": 0.5084556937217712, "learning_rate": 8.442882418044202e-05, "loss": 0.023209132254123688, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.02348, "step": 159, "tokens/total": 5513200, "tokens/train_per_sec_per_gpu": 28.87, "tokens/trainable": 72726} +{"epoch": 0.625, "grad_norm": 1.22331964969635, "learning_rate": 8.420860333495179e-05, "loss": 0.013725357130169868, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01382, "step": 160, "tokens/total": 5547120, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 73148} +{"epoch": 0.62890625, "grad_norm": 0.8818821907043457, "learning_rate": 8.398716700025208e-05, "loss": 0.017142701894044876, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.01729, "step": 161, "tokens/total": 5580336, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 73600} +{"epoch": 0.6328125, "grad_norm": 0.7221404910087585, "learning_rate": 8.376452439121266e-05, "loss": 0.011650539934635162, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.01172, "step": 162, "tokens/total": 5610736, "tokens/train_per_sec_per_gpu": 31.53, "tokens/trainable": 74068} +{"epoch": 0.63671875, "grad_norm": 0.8225802183151245, "learning_rate": 8.354068477290124e-05, "loss": 0.01369742676615715, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01379, "step": 163, "tokens/total": 5645040, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 74527} +{"epoch": 0.640625, "grad_norm": 0.4379892647266388, "learning_rate": 8.331565746019807e-05, "loss": 0.01064296904951334, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.0107, "step": 164, "tokens/total": 5676944, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 74992} +{"epoch": 0.64453125, "grad_norm": 0.6678451299667358, "learning_rate": 8.308945181740812e-05, "loss": 0.01863613910973072, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01881, "step": 165, "tokens/total": 5711728, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 75442} +{"epoch": 0.6484375, "grad_norm": 0.2177627682685852, "learning_rate": 8.286207725787153e-05, "loss": 0.0061256736516952515, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00614, "step": 166, "tokens/total": 5748656, "tokens/train_per_sec_per_gpu": 22.22, "tokens/trainable": 75887} +{"epoch": 0.65234375, "grad_norm": 14.120706558227539, "learning_rate": 8.263354324357182e-05, "loss": 0.03979087620973587, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.04059, "step": 167, "tokens/total": 5783376, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 76331} +{"epoch": 0.65625, "grad_norm": 0.17784035205841064, "learning_rate": 8.240385928474219e-05, "loss": 0.0045899502001702785, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.0046, "step": 168, "tokens/total": 5816512, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 76745} +{"epoch": 0.66015625, "grad_norm": 0.26492419838905334, "learning_rate": 8.217303493946967e-05, "loss": 0.006612748373299837, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00663, "step": 169, "tokens/total": 5850944, "tokens/train_per_sec_per_gpu": 25.86, "tokens/trainable": 77201} +{"epoch": 0.6640625, "grad_norm": 0.4899996519088745, "learning_rate": 8.194107981329746e-05, "loss": 0.013175439089536667, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01326, "step": 170, "tokens/total": 5888912, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 77655} +{"epoch": 0.66796875, "grad_norm": 0.11298198252916336, "learning_rate": 8.170800355882518e-05, "loss": 0.0029956516809761524, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.003, "step": 171, "tokens/total": 5923136, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 78122} +{"epoch": 0.671875, "grad_norm": 1.420280933380127, "learning_rate": 8.147381587530713e-05, "loss": 0.022452279925346375, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.02271, "step": 172, "tokens/total": 5956944, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 78576} +{"epoch": 0.67578125, "grad_norm": 0.5318289399147034, "learning_rate": 8.123852650824877e-05, "loss": 0.02029445394873619, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0205, "step": 173, "tokens/total": 5990976, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 79059} +{"epoch": 0.6796875, "grad_norm": 1.4191502332687378, "learning_rate": 8.100214524900103e-05, "loss": 0.022008134052157402, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.02225, "step": 174, "tokens/total": 6026368, "tokens/train_per_sec_per_gpu": 21.95, "tokens/trainable": 79498} +{"epoch": 0.68359375, "grad_norm": 0.3698967695236206, "learning_rate": 8.076468193435301e-05, "loss": 0.012148858979344368, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.01222, "step": 175, "tokens/total": 6057760, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 79922} +{"epoch": 0.6875, "grad_norm": 0.5445346832275391, "learning_rate": 8.052614644612253e-05, "loss": 0.008289042860269547, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00832, "step": 176, "tokens/total": 6095024, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 80383} +{"epoch": 0.69140625, "grad_norm": 0.10250675678253174, "learning_rate": 8.028654871074489e-05, "loss": 0.002193860709667206, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0022, "step": 177, "tokens/total": 6129712, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 80824} +{"epoch": 0.6953125, "grad_norm": 0.2638148367404938, "learning_rate": 8.004589869885986e-05, "loss": 0.00565145630389452, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00567, "step": 178, "tokens/total": 6161824, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 81243} +{"epoch": 0.69921875, "grad_norm": 0.7134056687355042, "learning_rate": 7.980420642489674e-05, "loss": 0.01627601496875286, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01641, "step": 179, "tokens/total": 6200464, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 81716} +{"epoch": 0.703125, "grad_norm": 0.643598198890686, "learning_rate": 7.95614819466576e-05, "loss": 0.003897574497386813, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00391, "step": 180, "tokens/total": 6234416, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 82181} +{"epoch": 0.70703125, "grad_norm": 0.12465423345565796, "learning_rate": 7.931773536489872e-05, "loss": 0.002978418953716755, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00298, "step": 181, "tokens/total": 6268608, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 82626} +{"epoch": 0.7109375, "grad_norm": 0.6406522393226624, "learning_rate": 7.907297682291035e-05, "loss": 0.008020609617233276, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00805, "step": 182, "tokens/total": 6303792, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 83089} +{"epoch": 0.71484375, "grad_norm": 0.23087327182292938, "learning_rate": 7.882721650609442e-05, "loss": 0.004715263843536377, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00473, "step": 183, "tokens/total": 6343664, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 83590} +{"epoch": 0.71875, "grad_norm": 0.35192734003067017, "learning_rate": 7.85804646415409e-05, "loss": 0.01460699550807476, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01471, "step": 184, "tokens/total": 6378848, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 84046} +{"epoch": 0.72265625, "grad_norm": 0.07029957324266434, "learning_rate": 7.833273149760207e-05, "loss": 0.0013019890757277608, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0013, "step": 185, "tokens/total": 6409968, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 84517} +{"epoch": 0.7265625, "grad_norm": 1.4329408407211304, "learning_rate": 7.808402738346527e-05, "loss": 0.011495120823383331, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01156, "step": 186, "tokens/total": 6448320, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 84974} +{"epoch": 0.73046875, "grad_norm": 0.06332936137914658, "learning_rate": 7.783436264872382e-05, "loss": 0.001191416522487998, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00119, "step": 187, "tokens/total": 6485440, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 85460} +{"epoch": 0.734375, "grad_norm": 0.23277878761291504, "learning_rate": 7.758374768294647e-05, "loss": 0.0072744302451610565, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.0073, "step": 188, "tokens/total": 6521216, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 85939} +{"epoch": 0.73828125, "grad_norm": 0.5697564482688904, "learning_rate": 7.733219291524489e-05, "loss": 0.01365069393068552, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01374, "step": 189, "tokens/total": 6554736, "tokens/train_per_sec_per_gpu": 28.18, "tokens/trainable": 86377} +{"epoch": 0.7421875, "grad_norm": 0.0874544084072113, "learning_rate": 7.707970881383977e-05, "loss": 0.0015968524385243654, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.0016, "step": 190, "tokens/total": 6586480, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 86834} +{"epoch": 0.74609375, "grad_norm": 0.1911919265985489, "learning_rate": 7.682630588562518e-05, "loss": 0.004160919692367315, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00417, "step": 191, "tokens/total": 6621536, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 87265} +{"epoch": 0.75, "grad_norm": 0.15456274151802063, "learning_rate": 7.657199467573129e-05, "loss": 0.005261835176497698, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00528, "step": 192, "tokens/total": 6655248, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 87747} +{"epoch": 0.75390625, "grad_norm": 0.2914696931838989, "learning_rate": 7.631678576708561e-05, "loss": 0.006347254849970341, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00637, "step": 193, "tokens/total": 6688960, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 88239} +{"epoch": 0.7578125, "grad_norm": 0.20387475192546844, "learning_rate": 7.606068977997255e-05, "loss": 0.008660852909088135, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0087, "step": 194, "tokens/total": 6726256, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 88718} +{"epoch": 0.76171875, "grad_norm": 0.3249609172344208, "learning_rate": 7.580371737159148e-05, "loss": 0.010195216163992882, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01025, "step": 195, "tokens/total": 6761984, "tokens/train_per_sec_per_gpu": 22.81, "tokens/trainable": 89129} +{"epoch": 0.765625, "grad_norm": 0.3592952489852905, "learning_rate": 7.554587923561324e-05, "loss": 0.004051889292895794, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00406, "step": 196, "tokens/total": 6796640, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 89567} +{"epoch": 0.76953125, "grad_norm": 0.44899484515190125, "learning_rate": 7.528718610173511e-05, "loss": 0.01202507596462965, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.0121, "step": 197, "tokens/total": 6831840, "tokens/train_per_sec_per_gpu": 22.59, "tokens/trainable": 89988} +{"epoch": 0.7734375, "grad_norm": 0.23219898343086243, "learning_rate": 7.502764873523431e-05, "loss": 0.0031681819818913937, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00317, "step": 198, "tokens/total": 6866128, "tokens/train_per_sec_per_gpu": 27.87, "tokens/trainable": 90434} +{"epoch": 0.77734375, "grad_norm": 0.32262423634529114, "learning_rate": 7.476727793652011e-05, "loss": 0.009265556000173092, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00931, "step": 199, "tokens/total": 6898960, "tokens/train_per_sec_per_gpu": 27.88, "tokens/trainable": 90913} +{"epoch": 0.78125, "grad_norm": 0.3987630307674408, "learning_rate": 7.450608454068415e-05, "loss": 0.01542721688747406, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01555, "step": 200, "tokens/total": 6933312, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 91355} +{"epoch": 0.78515625, "grad_norm": 0.10581964254379272, "learning_rate": 7.424407941704987e-05, "loss": 0.001808317843824625, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00181, "step": 201, "tokens/total": 6970016, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 91856} +{"epoch": 0.7890625, "grad_norm": 0.4351418614387512, "learning_rate": 7.398127346871986e-05, "loss": 0.009910037741065025, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00996, "step": 202, "tokens/total": 7006736, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 92311} +{"epoch": 0.79296875, "grad_norm": 0.1459161937236786, "learning_rate": 7.371767763212238e-05, "loss": 0.002789377234876156, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00279, "step": 203, "tokens/total": 7041728, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 92764} +{"epoch": 0.796875, "grad_norm": 0.2360151708126068, "learning_rate": 7.345330287655617e-05, "loss": 0.004407418426126242, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00442, "step": 204, "tokens/total": 7079792, "tokens/train_per_sec_per_gpu": 22.57, "tokens/trainable": 93227} +{"epoch": 0.80078125, "grad_norm": 0.8508720993995667, "learning_rate": 7.31881602037339e-05, "loss": 0.020570620894432068, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.02078, "step": 205, "tokens/total": 7111824, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 93675} +{"epoch": 0.8046875, "grad_norm": 0.4014439582824707, "learning_rate": 7.29222606473245e-05, "loss": 0.012443098239600658, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.01252, "step": 206, "tokens/total": 7144896, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 94120} +{"epoch": 0.80859375, "grad_norm": 0.13838942348957062, "learning_rate": 7.265561527249383e-05, "loss": 0.0016852119006216526, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00169, "step": 207, "tokens/total": 7178944, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 94557} +{"epoch": 0.8125, "grad_norm": 0.33806172013282776, "learning_rate": 7.238823517544436e-05, "loss": 0.00551880057901144, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00553, "step": 208, "tokens/total": 7215840, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 95035} +{"epoch": 0.81640625, "grad_norm": 0.2322305291891098, "learning_rate": 7.212013148295333e-05, "loss": 0.008334163576364517, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00837, "step": 209, "tokens/total": 7249952, "tokens/train_per_sec_per_gpu": 29.56, "tokens/trainable": 95517} +{"epoch": 0.8203125, "grad_norm": 0.19350311160087585, "learning_rate": 7.185131535190975e-05, "loss": 0.00213428121060133, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00214, "step": 210, "tokens/total": 7282352, "tokens/train_per_sec_per_gpu": 29.77, "tokens/trainable": 96026} +{"epoch": 0.82421875, "grad_norm": 0.06516356766223907, "learning_rate": 7.158179796885005e-05, "loss": 0.0009029427892528474, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0009, "step": 211, "tokens/total": 7316768, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 96477} +{"epoch": 0.828125, "grad_norm": 0.06259214133024216, "learning_rate": 7.131159054949273e-05, "loss": 0.0017425204860046506, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00174, "step": 212, "tokens/total": 7355168, "tokens/train_per_sec_per_gpu": 22.23, "tokens/trainable": 96951} +{"epoch": 0.83203125, "grad_norm": 0.7032011151313782, "learning_rate": 7.104070433827139e-05, "loss": 0.018178537487983704, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01834, "step": 213, "tokens/total": 7390720, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 97350} +{"epoch": 0.8359375, "grad_norm": 0.11332012712955475, "learning_rate": 7.076915060786705e-05, "loss": 0.0015845944872125983, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00159, "step": 214, "tokens/total": 7423616, "tokens/train_per_sec_per_gpu": 24.12, "tokens/trainable": 97792} +{"epoch": 0.83984375, "grad_norm": 0.23806661367416382, "learning_rate": 7.049694065873882e-05, "loss": 0.0053077582269907, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00532, "step": 215, "tokens/total": 7458352, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 98257} +{"epoch": 0.84375, "grad_norm": 0.4767276346683502, "learning_rate": 7.022408581865382e-05, "loss": 0.00873522274196148, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00877, "step": 216, "tokens/total": 7492672, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 98731} +{"epoch": 0.84765625, "grad_norm": 0.020585667341947556, "learning_rate": 6.99505974422157e-05, "loss": 0.0002509994083084166, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00025, "step": 217, "tokens/total": 7526832, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 99212} +{"epoch": 0.8515625, "grad_norm": 0.017499815672636032, "learning_rate": 6.967648691039213e-05, "loss": 0.0002635435666888952, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00026, "step": 218, "tokens/total": 7558944, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 99654} +{"epoch": 0.85546875, "grad_norm": 0.10180216282606125, "learning_rate": 6.940176563004123e-05, "loss": 0.0013638510135933757, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00136, "step": 219, "tokens/total": 7593120, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 100086} +{"epoch": 0.859375, "grad_norm": 0.2400251030921936, "learning_rate": 6.912644503343682e-05, "loss": 0.004608837887644768, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00462, "step": 220, "tokens/total": 7630592, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 100524} +{"epoch": 0.86328125, "grad_norm": 0.12966059148311615, "learning_rate": 6.885053657779273e-05, "loss": 0.0024217062164098024, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00242, "step": 221, "tokens/total": 7662384, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 100996} +{"epoch": 0.8671875, "grad_norm": 0.09719040244817734, "learning_rate": 6.857405174478604e-05, "loss": 0.0007005315274000168, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0007, "step": 222, "tokens/total": 7697152, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 101449} +{"epoch": 0.87109375, "grad_norm": 0.5149168372154236, "learning_rate": 6.82970020400792e-05, "loss": 0.02424125000834465, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02454, "step": 223, "tokens/total": 7734624, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 101905} +{"epoch": 0.875, "grad_norm": 0.3945418894290924, "learning_rate": 6.801939899284132e-05, "loss": 0.0034464166965335608, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00345, "step": 224, "tokens/total": 7770848, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 102411} +{"epoch": 0.87890625, "grad_norm": 0.07106346637010574, "learning_rate": 6.774125415526827e-05, "loss": 0.0011872659670189023, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00119, "step": 225, "tokens/total": 7803904, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 102882} +{"epoch": 0.8828125, "grad_norm": 0.048297300934791565, "learning_rate": 6.746257910210214e-05, "loss": 0.0005640499293804169, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00056, "step": 226, "tokens/total": 7837088, "tokens/train_per_sec_per_gpu": 27.79, "tokens/trainable": 103332} +{"epoch": 0.88671875, "grad_norm": 0.36240431666374207, "learning_rate": 6.718338543014937e-05, "loss": 0.006246998906135559, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00627, "step": 227, "tokens/total": 7867360, "tokens/train_per_sec_per_gpu": 31.67, "tokens/trainable": 103790} +{"epoch": 0.890625, "grad_norm": 0.2484816610813141, "learning_rate": 6.69036847577983e-05, "loss": 0.007445122580975294, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00747, "step": 228, "tokens/total": 7901280, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 104246} +{"epoch": 0.89453125, "grad_norm": 0.130929633975029, "learning_rate": 6.662348872453553e-05, "loss": 0.0036309738643467426, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00364, "step": 229, "tokens/total": 7937536, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 104707} +{"epoch": 0.8984375, "grad_norm": 0.059730350971221924, "learning_rate": 6.63428089904618e-05, "loss": 0.0007439877954311669, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00074, "step": 230, "tokens/total": 7976544, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 105167} +{"epoch": 0.90234375, "grad_norm": 0.06520292907953262, "learning_rate": 6.60616572358065e-05, "loss": 0.000981691526249051, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00098, "step": 231, "tokens/total": 8011008, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 105576} +{"epoch": 0.90625, "grad_norm": 0.029045991599559784, "learning_rate": 6.578004516044172e-05, "loss": 0.00046227191342040896, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00046, "step": 232, "tokens/total": 8046784, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 106043} +{"epoch": 0.91015625, "grad_norm": 0.3925827741622925, "learning_rate": 6.549798448339548e-05, "loss": 0.008806281723082066, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00885, "step": 233, "tokens/total": 8084048, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 106506} +{"epoch": 0.9140625, "grad_norm": 0.046352408826351166, "learning_rate": 6.521548694236384e-05, "loss": 0.0008021983085200191, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0008, "step": 234, "tokens/total": 8121248, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 106951} +{"epoch": 0.91796875, "grad_norm": 0.3981564939022064, "learning_rate": 6.493256429322259e-05, "loss": 0.011910950765013695, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01198, "step": 235, "tokens/total": 8157504, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 107382} +{"epoch": 0.921875, "grad_norm": 0.6587612628936768, "learning_rate": 6.464922830953799e-05, "loss": 0.016765015199780464, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01691, "step": 236, "tokens/total": 8196368, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 107814} +{"epoch": 0.92578125, "grad_norm": 0.6953209042549133, "learning_rate": 6.436549078207688e-05, "loss": 0.027788398787379265, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.02818, "step": 237, "tokens/total": 8229376, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 108274} +{"epoch": 0.9296875, "grad_norm": 0.294974148273468, "learning_rate": 6.408136351831592e-05, "loss": 0.009512309916317463, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00956, "step": 238, "tokens/total": 8261792, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 108714} +{"epoch": 0.93359375, "grad_norm": 0.21436375379562378, "learning_rate": 6.379685834195036e-05, "loss": 0.008363666944205761, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0084, "step": 239, "tokens/total": 8301216, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 109220} +{"epoch": 0.9375, "grad_norm": 0.12307509034872055, "learning_rate": 6.351198709240186e-05, "loss": 0.0028593428432941437, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00286, "step": 240, "tokens/total": 8341152, "tokens/train_per_sec_per_gpu": 20.82, "tokens/trainable": 109672} +{"epoch": 0.94140625, "grad_norm": 0.14444199204444885, "learning_rate": 6.32267616243259e-05, "loss": 0.006276214029639959, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0063, "step": 241, "tokens/total": 8373952, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 110135} +{"epoch": 0.9453125, "grad_norm": 0.11798764020204544, "learning_rate": 6.294119380711849e-05, "loss": 0.002936465200036764, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00294, "step": 242, "tokens/total": 8408240, "tokens/train_per_sec_per_gpu": 23.69, "tokens/trainable": 110563} +{"epoch": 0.94921875, "grad_norm": 0.3127708435058594, "learning_rate": 6.265529552442209e-05, "loss": 0.011000261642038822, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01106, "step": 243, "tokens/total": 8443344, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 111049} +{"epoch": 0.953125, "grad_norm": 0.2767637372016907, "learning_rate": 6.236907867363127e-05, "loss": 0.01025468297302723, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.01031, "step": 244, "tokens/total": 8477920, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 111495} +{"epoch": 0.95703125, "grad_norm": 0.07945647835731506, "learning_rate": 6.208255516539749e-05, "loss": 0.002427616622298956, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00243, "step": 245, "tokens/total": 8510240, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 111968} +{"epoch": 0.9609375, "grad_norm": 0.14855168759822845, "learning_rate": 6.179573692313344e-05, "loss": 0.006795767229050398, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00682, "step": 246, "tokens/total": 8547024, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 112416} +{"epoch": 0.96484375, "grad_norm": 0.11505711078643799, "learning_rate": 6.150863588251694e-05, "loss": 0.004089138936251402, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.0041, "step": 247, "tokens/total": 8586912, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 112882} +{"epoch": 0.96875, "grad_norm": 0.14159715175628662, "learning_rate": 6.122126399099419e-05, "loss": 0.004983797669410706, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.005, "step": 248, "tokens/total": 8626304, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 113390} +{"epoch": 0.97265625, "grad_norm": 0.354290246963501, "learning_rate": 6.0933633207282615e-05, "loss": 0.0178501196205616, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01801, "step": 249, "tokens/total": 8662736, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 113904} +{"epoch": 0.9765625, "grad_norm": 0.17286135256290436, "learning_rate": 6.064575550087316e-05, "loss": 0.006355185527354479, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00638, "step": 250, "tokens/total": 8699376, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 114351} +{"epoch": 0.98046875, "grad_norm": 0.2868482172489166, "learning_rate": 6.0357642851532245e-05, "loss": 0.006921407766640186, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00695, "step": 251, "tokens/total": 8736064, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 114842} +{"epoch": 0.984375, "grad_norm": 0.812897801399231, "learning_rate": 6.0069307248803294e-05, "loss": 0.008885964751243591, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00893, "step": 252, "tokens/total": 8771424, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 115263} +{"epoch": 0.98828125, "grad_norm": 0.1017707884311676, "learning_rate": 5.9780760691507635e-05, "loss": 0.0028922755736857653, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.0029, "step": 253, "tokens/total": 8803168, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 115703} +{"epoch": 0.9921875, "grad_norm": 0.15853576362133026, "learning_rate": 5.9492015187245334e-05, "loss": 0.0014440969098359346, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00145, "step": 254, "tokens/total": 8840912, "tokens/train_per_sec_per_gpu": 22.99, "tokens/trainable": 116157} +{"epoch": 0.99609375, "grad_norm": 0.35228434205055237, "learning_rate": 5.920308275189541e-05, "loss": 0.00414564274251461, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00415, "step": 255, "tokens/total": 8873184, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 116567} +{"epoch": 1.0, "grad_norm": 0.2760758101940155, "learning_rate": 5.8913975409115874e-05, "loss": 0.015042006969451904, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01516, "step": 256, "tokens/total": 8906544, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 117030} +{"epoch": 1.00390625, "grad_norm": 0.16870927810668945, "learning_rate": 5.8624705189843395e-05, "loss": 0.00262360367923975, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00263, "step": 257, "tokens/total": 8943760, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 117517} +{"epoch": 1.0078125, "grad_norm": 1.043556809425354, "learning_rate": 5.833528413179249e-05, "loss": 0.00224484084174037, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00225, "step": 258, "tokens/total": 8980256, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 118015} +{"epoch": 1.01171875, "grad_norm": 0.3802191913127899, "learning_rate": 5.80457242789548e-05, "loss": 0.00670089665800333, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00672, "step": 259, "tokens/total": 9020240, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 118520} +{"epoch": 1.015625, "grad_norm": 0.20763686299324036, "learning_rate": 5.77560376810977e-05, "loss": 0.002276776358485222, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00228, "step": 260, "tokens/total": 9053888, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 118992} +{"epoch": 1.01953125, "grad_norm": 0.027431020513176918, "learning_rate": 5.7466236393263005e-05, "loss": 0.000203912757569924, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0002, "step": 261, "tokens/total": 9088960, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 119438} +{"epoch": 1.0234375, "grad_norm": 0.009590369649231434, "learning_rate": 5.717633247526522e-05, "loss": 0.000171850057085976, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00017, "step": 262, "tokens/total": 9120256, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 119881} +{"epoch": 1.02734375, "grad_norm": 0.04097393527626991, "learning_rate": 5.688633799118971e-05, "loss": 0.0005390951409935951, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00054, "step": 263, "tokens/total": 9152336, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 120285} +{"epoch": 1.03125, "grad_norm": 0.17443467676639557, "learning_rate": 5.659626500889066e-05, "loss": 0.0015055584954097867, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00151, "step": 264, "tokens/total": 9186208, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 120755} +{"epoch": 1.03515625, "grad_norm": 0.020241623744368553, "learning_rate": 5.6306125599488905e-05, "loss": 0.00021775941422674805, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00022, "step": 265, "tokens/total": 9216768, "tokens/train_per_sec_per_gpu": 31.79, "tokens/trainable": 121194} +{"epoch": 1.0390625, "grad_norm": 0.7014346122741699, "learning_rate": 5.601593183686955e-05, "loss": 0.012853143736720085, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01294, "step": 266, "tokens/total": 9254032, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 121670} +{"epoch": 1.04296875, "grad_norm": 0.3869282603263855, "learning_rate": 5.572569579717961e-05, "loss": 0.007236198056489229, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00726, "step": 267, "tokens/total": 9289552, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 122142} +{"epoch": 1.046875, "grad_norm": 0.027126142755150795, "learning_rate": 5.543542955832538e-05, "loss": 0.00027674733428284526, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00028, "step": 268, "tokens/total": 9329312, "tokens/train_per_sec_per_gpu": 21.49, "tokens/trainable": 122585} +{"epoch": 1.05078125, "grad_norm": 0.39046168327331543, "learning_rate": 5.514514519946986e-05, "loss": 0.0019334092503413558, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00194, "step": 269, "tokens/total": 9369136, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 123091} +{"epoch": 1.0546875, "grad_norm": 0.01090212818235159, "learning_rate": 5.485485480053015e-05, "loss": 0.00016996482736431062, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00017, "step": 270, "tokens/total": 9407360, "tokens/train_per_sec_per_gpu": 21.73, "tokens/trainable": 123505} +{"epoch": 1.05859375, "grad_norm": 0.4262508749961853, "learning_rate": 5.4564570441674645e-05, "loss": 0.008472651243209839, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00851, "step": 271, "tokens/total": 9440624, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 123953} +{"epoch": 1.0625, "grad_norm": 0.5785030722618103, "learning_rate": 5.42743042028204e-05, "loss": 0.00473802350461483, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00475, "step": 272, "tokens/total": 9477120, "tokens/train_per_sec_per_gpu": 20.75, "tokens/trainable": 124400} +{"epoch": 1.06640625, "grad_norm": 0.28414204716682434, "learning_rate": 5.3984068163130464e-05, "loss": 0.0029024812392890453, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00291, "step": 273, "tokens/total": 9510832, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 124870} +{"epoch": 1.0703125, "grad_norm": 0.46413516998291016, "learning_rate": 5.369387440051111e-05, "loss": 0.008597622625529766, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00863, "step": 274, "tokens/total": 9544368, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 125333} +{"epoch": 1.07421875, "grad_norm": 0.027399683371186256, "learning_rate": 5.340373499110935e-05, "loss": 0.00044394045835360885, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00044, "step": 275, "tokens/total": 9580240, "tokens/train_per_sec_per_gpu": 27.07, "tokens/trainable": 125834} +{"epoch": 1.078125, "grad_norm": 0.004039996303617954, "learning_rate": 5.3113662008810304e-05, "loss": 8.556821558158845e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00009, "step": 276, "tokens/total": 9620256, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 126316} +{"epoch": 1.08203125, "grad_norm": 0.030507536605000496, "learning_rate": 5.282366752473479e-05, "loss": 0.00041544815758243203, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00042, "step": 277, "tokens/total": 9657264, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 126798} +{"epoch": 1.0859375, "grad_norm": 0.03944265842437744, "learning_rate": 5.2533763606737005e-05, "loss": 0.0005415845080278814, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00054, "step": 278, "tokens/total": 9696784, "tokens/train_per_sec_per_gpu": 19.18, "tokens/trainable": 127223} +{"epoch": 1.08984375, "grad_norm": 0.07566946744918823, "learning_rate": 5.224396231890232e-05, "loss": 0.00079974380787462, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.0008, "step": 279, "tokens/total": 9728256, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 127672} +{"epoch": 1.09375, "grad_norm": 0.2526216506958008, "learning_rate": 5.195427572104522e-05, "loss": 0.004667969886213541, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00468, "step": 280, "tokens/total": 9762688, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 128116} +{"epoch": 1.09765625, "grad_norm": 0.2070956528186798, "learning_rate": 5.166471586820751e-05, "loss": 0.00282917614094913, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00283, "step": 281, "tokens/total": 9799296, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 128589} +{"epoch": 1.1015625, "grad_norm": 0.7067859768867493, "learning_rate": 5.1375294810156615e-05, "loss": 0.017330065369606018, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01748, "step": 282, "tokens/total": 9834128, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 129036} +{"epoch": 1.10546875, "grad_norm": 0.04107745364308357, "learning_rate": 5.1086024590884144e-05, "loss": 0.0005860815872438252, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00059, "step": 283, "tokens/total": 9871536, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 129485} +{"epoch": 1.109375, "grad_norm": 0.07550673931837082, "learning_rate": 5.079691724810461e-05, "loss": 0.0012183313956484199, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00122, "step": 284, "tokens/total": 9907712, "tokens/train_per_sec_per_gpu": 21.66, "tokens/trainable": 129920} +{"epoch": 1.11328125, "grad_norm": 0.005032413173466921, "learning_rate": 5.0507984812754684e-05, "loss": 0.00011856925266329199, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00012, "step": 285, "tokens/total": 9941152, "tokens/train_per_sec_per_gpu": 28.57, "tokens/trainable": 130417} +{"epoch": 1.1171875, "grad_norm": 0.06184754520654678, "learning_rate": 5.021923930849237e-05, "loss": 0.0008782853838056326, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00088, "step": 286, "tokens/total": 9975472, "tokens/train_per_sec_per_gpu": 27.75, "tokens/trainable": 130903} +{"epoch": 1.12109375, "grad_norm": 0.11799892038106918, "learning_rate": 4.99306927511967e-05, "loss": 0.0015043567400425673, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00151, "step": 287, "tokens/total": 10009280, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 131343} +{"epoch": 1.125, "grad_norm": 0.027491239830851555, "learning_rate": 4.964235714846775e-05, "loss": 0.00046223518438637257, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00046, "step": 288, "tokens/total": 10041584, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 131763} +{"epoch": 1.12890625, "grad_norm": 0.01567123457789421, "learning_rate": 4.9354244499126866e-05, "loss": 0.00019140726362820715, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00019, "step": 289, "tokens/total": 10073312, "tokens/train_per_sec_per_gpu": 29.18, "tokens/trainable": 132229} +{"epoch": 1.1328125, "grad_norm": 0.13266998529434204, "learning_rate": 4.90663667927174e-05, "loss": 0.0012123656924813986, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00121, "step": 290, "tokens/total": 10106160, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 132678} +{"epoch": 1.13671875, "grad_norm": 0.009990338236093521, "learning_rate": 4.877873600900581e-05, "loss": 0.00013909710105508566, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00014, "step": 291, "tokens/total": 10142192, "tokens/train_per_sec_per_gpu": 21.61, "tokens/trainable": 133136} +{"epoch": 1.140625, "grad_norm": 0.19812455773353577, "learning_rate": 4.849136411748306e-05, "loss": 0.0011534905061125755, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00115, "step": 292, "tokens/total": 10179776, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 133608} +{"epoch": 1.14453125, "grad_norm": 0.17589595913887024, "learning_rate": 4.8204263076866574e-05, "loss": 0.004077166318893433, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00409, "step": 293, "tokens/total": 10215424, "tokens/train_per_sec_per_gpu": 29.48, "tokens/trainable": 134108} +{"epoch": 1.1484375, "grad_norm": 0.004372004419565201, "learning_rate": 4.791744483460251e-05, "loss": 8.58479761518538e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00009, "step": 294, "tokens/total": 10249632, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 134541} +{"epoch": 1.15234375, "grad_norm": 0.001756952260620892, "learning_rate": 4.7630921326368736e-05, "loss": 4.446303501026705e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00004, "step": 295, "tokens/total": 10288272, "tokens/train_per_sec_per_gpu": 19.95, "tokens/trainable": 134966} +{"epoch": 1.15625, "grad_norm": 0.335573673248291, "learning_rate": 4.7344704475577916e-05, "loss": 0.005048254504799843, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00506, "step": 296, "tokens/total": 10318160, "tokens/train_per_sec_per_gpu": 30.16, "tokens/trainable": 135402} +{"epoch": 1.16015625, "grad_norm": 0.7681025266647339, "learning_rate": 4.705880619288153e-05, "loss": 0.006290112622082233, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00631, "step": 297, "tokens/total": 10354304, "tokens/train_per_sec_per_gpu": 21.89, "tokens/trainable": 135804} +{"epoch": 1.1640625, "grad_norm": 0.008123784326016903, "learning_rate": 4.677323837567412e-05, "loss": 0.00018655734311323613, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00019, "step": 298, "tokens/total": 10386960, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 136231} +{"epoch": 1.16796875, "grad_norm": 0.04159111902117729, "learning_rate": 4.6488012907598146e-05, "loss": 8.622300083516166e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00009, "step": 299, "tokens/total": 10419712, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 136705} +{"epoch": 1.171875, "grad_norm": 0.008765353821218014, "learning_rate": 4.620314165804964e-05, "loss": 0.00018572533736005425, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00019, "step": 300, "tokens/total": 10455376, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 137178} +{"epoch": 1.17578125, "grad_norm": 0.2505803406238556, "learning_rate": 4.591863648168407e-05, "loss": 0.0018037431873381138, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00181, "step": 301, "tokens/total": 10489712, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 137643} +{"epoch": 1.1796875, "grad_norm": 0.028304776176810265, "learning_rate": 4.5634509217923135e-05, "loss": 0.0003735318314284086, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00037, "step": 302, "tokens/total": 10523056, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 138078} +{"epoch": 1.18359375, "grad_norm": 0.9593371152877808, "learning_rate": 4.535077169046201e-05, "loss": 0.010672494769096375, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01073, "step": 303, "tokens/total": 10559616, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 138515} +{"epoch": 1.1875, "grad_norm": 0.3550161123275757, "learning_rate": 4.506743570677743e-05, "loss": 0.0048037245869636536, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00482, "step": 304, "tokens/total": 10590208, "tokens/train_per_sec_per_gpu": 31.11, "tokens/trainable": 138948} +{"epoch": 1.19140625, "grad_norm": 0.1329122930765152, "learning_rate": 4.478451305763618e-05, "loss": 0.001264120452105999, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00126, "step": 305, "tokens/total": 10623344, "tokens/train_per_sec_per_gpu": 30.4, "tokens/trainable": 139408} +{"epoch": 1.1953125, "grad_norm": 0.005383720155805349, "learning_rate": 4.450201551660454e-05, "loss": 0.00010883378854487091, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00011, "step": 306, "tokens/total": 10655056, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 139840} +{"epoch": 1.19921875, "grad_norm": 0.360541969537735, "learning_rate": 4.4219954839558276e-05, "loss": 0.010587172582745552, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01064, "step": 307, "tokens/total": 10689744, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 140281} +{"epoch": 1.203125, "grad_norm": 0.26627716422080994, "learning_rate": 4.393834276419352e-05, "loss": 0.003391267266124487, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0034, "step": 308, "tokens/total": 10721456, "tokens/train_per_sec_per_gpu": 31.4, "tokens/trainable": 140776} +{"epoch": 1.20703125, "grad_norm": 0.013919971883296967, "learning_rate": 4.36571910095382e-05, "loss": 0.00011156620166730136, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00011, "step": 309, "tokens/total": 10759568, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 141228} +{"epoch": 1.2109375, "grad_norm": 0.1582929491996765, "learning_rate": 4.337651127546448e-05, "loss": 0.0034988224506378174, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0035, "step": 310, "tokens/total": 10795248, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 141679} +{"epoch": 1.21484375, "grad_norm": 0.16222676634788513, "learning_rate": 4.3096315242201736e-05, "loss": 0.0020528999157249928, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00206, "step": 311, "tokens/total": 10828208, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 142122} +{"epoch": 1.21875, "grad_norm": 0.02150947041809559, "learning_rate": 4.2816614569850635e-05, "loss": 0.0002162584278266877, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00022, "step": 312, "tokens/total": 10863984, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 142561} +{"epoch": 1.22265625, "grad_norm": 0.0878685712814331, "learning_rate": 4.2537420897897864e-05, "loss": 0.0013373036636039615, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00134, "step": 313, "tokens/total": 10899392, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 143046} +{"epoch": 1.2265625, "grad_norm": 0.0481707863509655, "learning_rate": 4.225874584473174e-05, "loss": 0.0004986703279428184, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.0005, "step": 314, "tokens/total": 10932224, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 143493} +{"epoch": 1.23046875, "grad_norm": 0.013401225209236145, "learning_rate": 4.19806010071587e-05, "loss": 0.00023170722124632448, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00023, "step": 315, "tokens/total": 10970096, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 143956} +{"epoch": 1.234375, "grad_norm": 0.3020864427089691, "learning_rate": 4.170299795992081e-05, "loss": 0.0074148159474134445, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00744, "step": 316, "tokens/total": 11003488, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 144411} +{"epoch": 1.23828125, "grad_norm": 0.09822997450828552, "learning_rate": 4.142594825521398e-05, "loss": 0.0012756327632814646, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00128, "step": 317, "tokens/total": 11042784, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 144892} +{"epoch": 1.2421875, "grad_norm": 0.023818673565983772, "learning_rate": 4.114946342220728e-05, "loss": 0.00025419803569093347, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00025, "step": 318, "tokens/total": 11078992, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 145339} +{"epoch": 1.24609375, "grad_norm": 0.13377732038497925, "learning_rate": 4.087355496656321e-05, "loss": 0.0015609429683536291, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00156, "step": 319, "tokens/total": 11113024, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 145811} +{"epoch": 1.25, "grad_norm": 0.09358290582895279, "learning_rate": 4.05982343699588e-05, "loss": 0.001358976005576551, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00136, "step": 320, "tokens/total": 11152240, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 146314} +{"epoch": 1.25390625, "grad_norm": 0.024258211255073547, "learning_rate": 4.0323513089607876e-05, "loss": 0.00031819017021916807, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00032, "step": 321, "tokens/total": 11187296, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 146781} +{"epoch": 1.2578125, "grad_norm": 0.1777718961238861, "learning_rate": 4.004940255778431e-05, "loss": 0.0010795829584822059, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00108, "step": 322, "tokens/total": 11225904, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 147225} +{"epoch": 1.26171875, "grad_norm": 0.637929379940033, "learning_rate": 3.977591418134619e-05, "loss": 0.0014261974720284343, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00143, "step": 323, "tokens/total": 11261056, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 147673} +{"epoch": 1.265625, "grad_norm": 0.09190244227647781, "learning_rate": 3.95030593412612e-05, "loss": 0.0013271612115204334, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00133, "step": 324, "tokens/total": 11296432, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 148103} +{"epoch": 1.26953125, "grad_norm": 0.013899151235818863, "learning_rate": 3.923084939213296e-05, "loss": 0.00014999281847849488, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00015, "step": 325, "tokens/total": 11327248, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 148535} +{"epoch": 1.2734375, "grad_norm": 0.23714616894721985, "learning_rate": 3.895929566172861e-05, "loss": 0.0022264006547629833, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00223, "step": 326, "tokens/total": 11363264, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 148999} +{"epoch": 1.27734375, "grad_norm": 0.15034598112106323, "learning_rate": 3.868840945050728e-05, "loss": 0.001604428282007575, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00161, "step": 327, "tokens/total": 11401776, "tokens/train_per_sec_per_gpu": 21.41, "tokens/trainable": 149431} +{"epoch": 1.28125, "grad_norm": 0.11734051257371902, "learning_rate": 3.841820203114995e-05, "loss": 0.0015709061408415437, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00157, "step": 328, "tokens/total": 11433520, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 149886} +{"epoch": 1.28515625, "grad_norm": 0.041013237088918686, "learning_rate": 3.814868464809027e-05, "loss": 0.0004517544584814459, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00045, "step": 329, "tokens/total": 11465376, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 150288} +{"epoch": 1.2890625, "grad_norm": 0.2482723891735077, "learning_rate": 3.787986851704667e-05, "loss": 0.0020060157403349876, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00201, "step": 330, "tokens/total": 11499120, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 150733} +{"epoch": 1.29296875, "grad_norm": 0.00670870253816247, "learning_rate": 3.7611764824555654e-05, "loss": 8.278960012830794e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00008, "step": 331, "tokens/total": 11531104, "tokens/train_per_sec_per_gpu": 30.73, "tokens/trainable": 151207} +{"epoch": 1.296875, "grad_norm": 0.14205874502658844, "learning_rate": 3.734438472750619e-05, "loss": 0.0013259898405522108, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00133, "step": 332, "tokens/total": 11571040, "tokens/train_per_sec_per_gpu": 24.51, "tokens/trainable": 151694} +{"epoch": 1.30078125, "grad_norm": 0.17401932179927826, "learning_rate": 3.707773935267552e-05, "loss": 0.0006437042029574513, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00064, "step": 333, "tokens/total": 11607952, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 152188} +{"epoch": 1.3046875, "grad_norm": 0.005255027208477259, "learning_rate": 3.68118397962661e-05, "loss": 4.1886873077601194e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00004, "step": 334, "tokens/total": 11642864, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 152596} +{"epoch": 1.30859375, "grad_norm": 0.23923513293266296, "learning_rate": 3.654669712344384e-05, "loss": 0.001604609307833016, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00161, "step": 335, "tokens/total": 11673904, "tokens/train_per_sec_per_gpu": 31.45, "tokens/trainable": 153052} +{"epoch": 1.3125, "grad_norm": 0.0062875025905668736, "learning_rate": 3.628232236787763e-05, "loss": 6.77796924719587e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00007, "step": 336, "tokens/total": 11709536, "tokens/train_per_sec_per_gpu": 22.55, "tokens/trainable": 153491} +{"epoch": 1.31640625, "grad_norm": 0.059719279408454895, "learning_rate": 3.6018726531280144e-05, "loss": 0.0002513603831175715, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00025, "step": 337, "tokens/total": 11747248, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 153983} +{"epoch": 1.3203125, "grad_norm": 0.39272549748420715, "learning_rate": 3.575592058295017e-05, "loss": 0.0002581291482783854, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00026, "step": 338, "tokens/total": 11781104, "tokens/train_per_sec_per_gpu": 28.55, "tokens/trainable": 154464} +{"epoch": 1.32421875, "grad_norm": 0.0009200222557410598, "learning_rate": 3.549391545931585e-05, "loss": 8.525092198397033e-06, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00001, "step": 339, "tokens/total": 11815712, "tokens/train_per_sec_per_gpu": 27.57, "tokens/trainable": 154924} +{"epoch": 1.328125, "grad_norm": 0.004991130903363228, "learning_rate": 3.5232722063479914e-05, "loss": 4.533741594059393e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00005, "step": 340, "tokens/total": 11849424, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 155373} +{"epoch": 1.33203125, "grad_norm": 0.004255872685462236, "learning_rate": 3.49723512647657e-05, "loss": 5.045306897955015e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00005, "step": 341, "tokens/total": 11888128, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 155873} +{"epoch": 1.3359375, "grad_norm": 0.12952496111392975, "learning_rate": 3.471281389826491e-05, "loss": 0.0009308229200541973, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00093, "step": 342, "tokens/total": 11919264, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 156278} +{"epoch": 1.33984375, "grad_norm": 0.005658402107656002, "learning_rate": 3.4454120764386764e-05, "loss": 4.178431845502928e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00004, "step": 343, "tokens/total": 11957456, "tokens/train_per_sec_per_gpu": 24.23, "tokens/trainable": 156733} +{"epoch": 1.34375, "grad_norm": 0.4221668243408203, "learning_rate": 3.4196282628408526e-05, "loss": 0.00512117613106966, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00513, "step": 344, "tokens/total": 11991696, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 157203} +{"epoch": 1.34765625, "grad_norm": 0.21650409698486328, "learning_rate": 3.3939310220027456e-05, "loss": 0.0019463851349428296, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00195, "step": 345, "tokens/total": 12025840, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 157707} +{"epoch": 1.3515625, "grad_norm": 0.0028265994042158127, "learning_rate": 3.3683214232914404e-05, "loss": 1.76117719092872e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00002, "step": 346, "tokens/total": 12061296, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 158180} +{"epoch": 1.35546875, "grad_norm": 0.019429558888077736, "learning_rate": 3.342800532426873e-05, "loss": 0.00017496541840955615, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00017, "step": 347, "tokens/total": 12097232, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 158646} +{"epoch": 1.359375, "grad_norm": 0.004642733372747898, "learning_rate": 3.317369411437484e-05, "loss": 5.0444083171896636e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00005, "step": 348, "tokens/total": 12133200, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 159115} +{"epoch": 1.36328125, "grad_norm": 0.781234622001648, "learning_rate": 3.292029118616024e-05, "loss": 0.004329551011323929, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00434, "step": 349, "tokens/total": 12166336, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 159538} +{"epoch": 1.3671875, "grad_norm": 0.10536090284585953, "learning_rate": 3.266780708475511e-05, "loss": 0.0009794221259653568, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00098, "step": 350, "tokens/total": 12201872, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 160016} +{"epoch": 1.37109375, "grad_norm": 0.0020283516496419907, "learning_rate": 3.241625231705354e-05, "loss": 1.9122355297440663e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00002, "step": 351, "tokens/total": 12238016, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 160475} +{"epoch": 1.375, "grad_norm": 0.12025554478168488, "learning_rate": 3.216563735127618e-05, "loss": 0.0008352678851224482, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00084, "step": 352, "tokens/total": 12273280, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 160952} +{"epoch": 1.37890625, "grad_norm": 0.8848488926887512, "learning_rate": 3.191597261653475e-05, "loss": 0.007823835127055645, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00785, "step": 353, "tokens/total": 12310096, "tokens/train_per_sec_per_gpu": 23.51, "tokens/trainable": 161420} +{"epoch": 1.3828125, "grad_norm": 10.04698657989502, "learning_rate": 3.166726850239794e-05, "loss": 0.0036431578919291496, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00365, "step": 354, "tokens/total": 12343136, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 161868} +{"epoch": 1.38671875, "grad_norm": 0.0010315482504665852, "learning_rate": 3.141953535845912e-05, "loss": 1.475249337090645e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00001, "step": 355, "tokens/total": 12374576, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 162318} +{"epoch": 1.390625, "grad_norm": 0.0013654843205586076, "learning_rate": 3.11727834939056e-05, "loss": 2.4058794224401936e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00002, "step": 356, "tokens/total": 12406896, "tokens/train_per_sec_per_gpu": 26.17, "tokens/trainable": 162789} +{"epoch": 1.39453125, "grad_norm": 0.28700461983680725, "learning_rate": 3.092702317708967e-05, "loss": 0.004793305415660143, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.0048, "step": 357, "tokens/total": 12440480, "tokens/train_per_sec_per_gpu": 28.74, "tokens/trainable": 163254} +{"epoch": 1.3984375, "grad_norm": 0.002572318771854043, "learning_rate": 3.0682264635101276e-05, "loss": 2.5353092496516183e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00003, "step": 358, "tokens/total": 12473232, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 163715} +{"epoch": 1.40234375, "grad_norm": 0.015000012703239918, "learning_rate": 3.0438518053342407e-05, "loss": 0.00011152267688885331, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00011, "step": 359, "tokens/total": 12512176, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 164197} +{"epoch": 1.40625, "grad_norm": 0.014959041960537434, "learning_rate": 3.0195793575103266e-05, "loss": 8.675039134686813e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00009, "step": 360, "tokens/total": 12551136, "tokens/train_per_sec_per_gpu": 21.17, "tokens/trainable": 164661} +{"epoch": 1.41015625, "grad_norm": 0.03769446909427643, "learning_rate": 2.9954101301140146e-05, "loss": 0.00028730701887980103, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00029, "step": 361, "tokens/total": 12586560, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.0049833147786557674, "learning_rate": 2.9713451289255123e-05, "loss": 7.274970266735181e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00007, "step": 362, "tokens/total": 12615936, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 165552} +{"epoch": 1.41796875, "grad_norm": 0.011645285412669182, "learning_rate": 2.9473853553877484e-05, "loss": 0.00010940147330984473, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00011, "step": 363, "tokens/total": 12650096, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 166026} +{"epoch": 1.421875, "grad_norm": 0.0018994236597791314, "learning_rate": 2.9235318065647e-05, "loss": 2.5692050257930532e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00003, "step": 364, "tokens/total": 12689808, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 166486} +{"epoch": 1.42578125, "grad_norm": 0.04375705122947693, "learning_rate": 2.8997854750998964e-05, "loss": 0.00030014163348823786, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.0003, "step": 365, "tokens/total": 12729328, "tokens/train_per_sec_per_gpu": 23.79, "tokens/trainable": 166959} +{"epoch": 1.4296875, "grad_norm": 0.009574740193784237, "learning_rate": 2.8761473491751258e-05, "loss": 3.869867214234546e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00004, "step": 366, "tokens/total": 12762576, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 167394} +{"epoch": 1.43359375, "grad_norm": 0.0018494944088160992, "learning_rate": 2.8526184124692883e-05, "loss": 2.8007703804178163e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00003, "step": 367, "tokens/total": 12798944, "tokens/train_per_sec_per_gpu": 20.28, "tokens/trainable": 167832} +{"epoch": 1.4375, "grad_norm": 0.032837241888046265, "learning_rate": 2.829199644117484e-05, "loss": 0.0002431349130347371, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00024, "step": 368, "tokens/total": 12832928, "tokens/train_per_sec_per_gpu": 24.22, "tokens/trainable": 168242} +{"epoch": 1.44140625, "grad_norm": 0.0356852263212204, "learning_rate": 2.8058920186702553e-05, "loss": 0.00013145152479410172, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00013, "step": 369, "tokens/total": 12870544, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 168715} +{"epoch": 1.4453125, "grad_norm": 0.01167068537324667, "learning_rate": 2.782696506053033e-05, "loss": 0.00012420908024068922, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00012, "step": 370, "tokens/total": 12908416, "tokens/train_per_sec_per_gpu": 22.28, "tokens/trainable": 169147} +{"epoch": 1.44921875, "grad_norm": 0.29082757234573364, "learning_rate": 2.7596140715257824e-05, "loss": 0.0032320385798811913, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00324, "step": 371, "tokens/total": 12943584, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 169628} +{"epoch": 1.453125, "grad_norm": 0.0664801225066185, "learning_rate": 2.7366456756428184e-05, "loss": 0.0004943436360917985, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00049, "step": 372, "tokens/total": 12978192, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 170085} +{"epoch": 1.45703125, "grad_norm": 0.09397958219051361, "learning_rate": 2.7137922742128486e-05, "loss": 0.0004196885274723172, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00042, "step": 373, "tokens/total": 13012720, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 170584} +{"epoch": 1.4609375, "grad_norm": 0.07453603297472, "learning_rate": 2.691054818259188e-05, "loss": 0.0008598973508924246, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00086, "step": 374, "tokens/total": 13047568, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 171058} +{"epoch": 1.46484375, "grad_norm": 0.1721239984035492, "learning_rate": 2.6684342539801933e-05, "loss": 0.0010824190685525537, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00108, "step": 375, "tokens/total": 13079088, "tokens/train_per_sec_per_gpu": 28.67, "tokens/trainable": 171518} +{"epoch": 1.46875, "grad_norm": 0.25899043679237366, "learning_rate": 2.645931522709877e-05, "loss": 0.0010054954327642918, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00101, "step": 376, "tokens/total": 13114480, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 172007} +{"epoch": 1.47265625, "grad_norm": 0.32763397693634033, "learning_rate": 2.6235475608787365e-05, "loss": 0.0021271901205182076, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00213, "step": 377, "tokens/total": 13150160, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 172468} +{"epoch": 1.4765625, "grad_norm": 0.14795000851154327, "learning_rate": 2.6012832999747916e-05, "loss": 0.0010855119908228517, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00109, "step": 378, "tokens/total": 13181280, "tokens/train_per_sec_per_gpu": 29.2, "tokens/trainable": 172923} +{"epoch": 1.48046875, "grad_norm": 0.4841530919075012, "learning_rate": 2.579139666504821e-05, "loss": 0.014943018555641174, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.01506, "step": 379, "tokens/total": 13221264, "tokens/train_per_sec_per_gpu": 22.09, "tokens/trainable": 173370} +{"epoch": 1.484375, "grad_norm": 0.023614175617694855, "learning_rate": 2.557117581955798e-05, "loss": 0.00020690543169621378, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00021, "step": 380, "tokens/total": 13255104, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 173837} +{"epoch": 1.48828125, "grad_norm": 0.027606088668107986, "learning_rate": 2.5352179627565532e-05, "loss": 0.00016978174971882254, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00017, "step": 381, "tokens/total": 13289856, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 174294} +{"epoch": 1.4921875, "grad_norm": 0.007501950487494469, "learning_rate": 2.5134417202396277e-05, "loss": 8.929081377573311e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00009, "step": 382, "tokens/total": 13323840, "tokens/train_per_sec_per_gpu": 23.85, "tokens/trainable": 174701} +{"epoch": 1.49609375, "grad_norm": 0.1021055281162262, "learning_rate": 2.491789760603361e-05, "loss": 0.00028551730792969465, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00029, "step": 383, "tokens/total": 13360256, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 175131} +{"epoch": 1.5, "grad_norm": 0.045718878507614136, "learning_rate": 2.4702629848741764e-05, "loss": 0.0005032605258747935, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0005, "step": 384, "tokens/total": 13394304, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 175586} +{"epoch": 1.50390625, "grad_norm": 0.64234459400177, "learning_rate": 2.4488622888690785e-05, "loss": 0.011786052957177162, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.01186, "step": 385, "tokens/total": 13427760, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 176026} +{"epoch": 1.5078125, "grad_norm": 0.014482767321169376, "learning_rate": 2.427588563158384e-05, "loss": 0.0001794650888768956, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00018, "step": 386, "tokens/total": 13462096, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 176512} +{"epoch": 1.51171875, "grad_norm": 0.0604175440967083, "learning_rate": 2.406442693028651e-05, "loss": 0.00046659144572913647, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00047, "step": 387, "tokens/total": 13494976, "tokens/train_per_sec_per_gpu": 24.76, "tokens/trainable": 176943} +{"epoch": 1.515625, "grad_norm": 0.05489639192819595, "learning_rate": 2.3854255584458547e-05, "loss": 0.0003972745034843683, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.0004, "step": 388, "tokens/total": 13533568, "tokens/train_per_sec_per_gpu": 21.5, "tokens/trainable": 177370} +{"epoch": 1.51953125, "grad_norm": 0.005990303121507168, "learning_rate": 2.3645380340187508e-05, "loss": 5.114184023113921e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00005, "step": 389, "tokens/total": 13569248, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 177851} +{"epoch": 1.5234375, "grad_norm": 0.28532594442367554, "learning_rate": 2.3437809889624914e-05, "loss": 0.004724485334008932, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00474, "step": 390, "tokens/total": 13601568, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 178310} +{"epoch": 1.52734375, "grad_norm": 0.005206988658756018, "learning_rate": 2.3231552870624487e-05, "loss": 7.50038743717596e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00008, "step": 391, "tokens/total": 13632576, "tokens/train_per_sec_per_gpu": 29.04, "tokens/trainable": 178736} +{"epoch": 1.53125, "grad_norm": 0.03151891753077507, "learning_rate": 2.3026617866382657e-05, "loss": 0.0004407903179526329, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00044, "step": 392, "tokens/total": 13665328, "tokens/train_per_sec_per_gpu": 29.74, "tokens/trainable": 179233} +{"epoch": 1.53515625, "grad_norm": 0.1928960084915161, "learning_rate": 2.2823013405081507e-05, "loss": 0.00233669625595212, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00234, "step": 393, "tokens/total": 13703408, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 179730} +{"epoch": 1.5390625, "grad_norm": 0.14893029630184174, "learning_rate": 2.2620747959533722e-05, "loss": 0.0014547251630574465, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00146, "step": 394, "tokens/total": 13736480, "tokens/train_per_sec_per_gpu": 30.54, "tokens/trainable": 180227} +{"epoch": 1.54296875, "grad_norm": 0.019966738298535347, "learning_rate": 2.2419829946830123e-05, "loss": 0.0001388439122820273, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00014, "step": 395, "tokens/total": 13772672, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 180687} +{"epoch": 1.546875, "grad_norm": 0.25807976722717285, "learning_rate": 2.2220267727989325e-05, "loss": 0.0019871697295457125, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00199, "step": 396, "tokens/total": 13806816, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 181141} +{"epoch": 1.55078125, "grad_norm": 0.18116147816181183, "learning_rate": 2.202206960760984e-05, "loss": 0.002870761789381504, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00287, "step": 397, "tokens/total": 13844960, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 181648} +{"epoch": 1.5546875, "grad_norm": 0.07355982065200806, "learning_rate": 2.182524383352446e-05, "loss": 0.0009820619598031044, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00098, "step": 398, "tokens/total": 13881120, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 182109} +{"epoch": 1.55859375, "grad_norm": 0.372219055891037, "learning_rate": 2.1629798596457056e-05, "loss": 0.015682725235819817, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01581, "step": 399, "tokens/total": 13913440, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 182559} +{"epoch": 1.5625, "grad_norm": 0.0016155141638591886, "learning_rate": 2.1435742029681725e-05, "loss": 2.7327860152581707e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00003, "step": 400, "tokens/total": 13945408, "tokens/train_per_sec_per_gpu": 27.53, "tokens/trainable": 182985} +{"epoch": 1.56640625, "grad_norm": 0.011626768857240677, "learning_rate": 2.124308220868431e-05, "loss": 0.00012487563071772456, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00012, "step": 401, "tokens/total": 13980800, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 183440} +{"epoch": 1.5703125, "grad_norm": 0.13166247308254242, "learning_rate": 2.105182715082638e-05, "loss": 0.0017556654056534171, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00176, "step": 402, "tokens/total": 14014400, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 183917} +{"epoch": 1.57421875, "grad_norm": 0.04118660092353821, "learning_rate": 2.0861984815011552e-05, "loss": 0.0003070707607548684, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00031, "step": 403, "tokens/total": 14049920, "tokens/train_per_sec_per_gpu": 22.46, "tokens/trainable": 184348} +{"epoch": 1.578125, "grad_norm": 0.010526592843234539, "learning_rate": 2.0673563101354323e-05, "loss": 0.00015115333371795714, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00015, "step": 404, "tokens/total": 14085200, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 184786} +{"epoch": 1.58203125, "grad_norm": 0.016778137534856796, "learning_rate": 2.0486569850851317e-05, "loss": 0.00017647151253186166, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00018, "step": 405, "tokens/total": 14120208, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 185249} +{"epoch": 1.5859375, "grad_norm": 0.19704371690750122, "learning_rate": 2.0301012845054956e-05, "loss": 0.002857377054169774, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00286, "step": 406, "tokens/total": 14158960, "tokens/train_per_sec_per_gpu": 21.27, "tokens/trainable": 185680} +{"epoch": 1.58984375, "grad_norm": 0.03820028156042099, "learning_rate": 2.011689980574966e-05, "loss": 0.0004195459187030792, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00042, "step": 407, "tokens/total": 14195264, "tokens/train_per_sec_per_gpu": 22.78, "tokens/trainable": 186100} +{"epoch": 1.59375, "grad_norm": 0.0729815661907196, "learning_rate": 1.993423839463052e-05, "loss": 0.00062619085656479, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00063, "step": 408, "tokens/total": 14228528, "tokens/train_per_sec_per_gpu": 31.69, "tokens/trainable": 186565} +{"epoch": 1.59765625, "grad_norm": 0.41507741808891296, "learning_rate": 1.975303621298445e-05, "loss": 0.002662857761606574, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00267, "step": 409, "tokens/total": 14265568, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 187017} +{"epoch": 1.6015625, "grad_norm": 0.024183206260204315, "learning_rate": 1.957330080137385e-05, "loss": 0.000362630991730839, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.00036, "step": 410, "tokens/total": 14296432, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 187468} +{"epoch": 1.60546875, "grad_norm": 0.10622778534889221, "learning_rate": 1.9395039639322864e-05, "loss": 0.0015962667530402541, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.0016, "step": 411, "tokens/total": 14332480, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 187961} +{"epoch": 1.609375, "grad_norm": 0.001539794378913939, "learning_rate": 1.9218260145006073e-05, "loss": 2.39009805227397e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00002, "step": 412, "tokens/total": 14367728, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 188447} +{"epoch": 1.61328125, "grad_norm": 0.13679459691047668, "learning_rate": 1.904296967493982e-05, "loss": 0.0023613842204213142, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00236, "step": 413, "tokens/total": 14399888, "tokens/train_per_sec_per_gpu": 29.42, "tokens/trainable": 188903} +{"epoch": 1.6171875, "grad_norm": 0.1202077567577362, "learning_rate": 1.8869175523676064e-05, "loss": 0.0017066169530153275, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00171, "step": 414, "tokens/total": 14436576, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 189398} +{"epoch": 1.62109375, "grad_norm": 0.011895768344402313, "learning_rate": 1.869688492349885e-05, "loss": 0.00011149967758683488, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00011, "step": 415, "tokens/total": 14470032, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 189850} +{"epoch": 1.625, "grad_norm": 0.17871327698230743, "learning_rate": 1.85261050441233e-05, "loss": 0.002293672412633896, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.0023, "step": 416, "tokens/total": 14505824, "tokens/train_per_sec_per_gpu": 24.25, "tokens/trainable": 190306} +{"epoch": 1.62890625, "grad_norm": 0.46320340037345886, "learning_rate": 1.8356842992397304e-05, "loss": 0.0140554653480649, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01415, "step": 417, "tokens/total": 14539344, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 190746} +{"epoch": 1.6328125, "grad_norm": 0.03403477370738983, "learning_rate": 1.8189105812005714e-05, "loss": 0.0005694678984582424, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00057, "step": 418, "tokens/total": 14576160, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 191156} +{"epoch": 1.63671875, "grad_norm": 0.04663674160838127, "learning_rate": 1.802290048317732e-05, "loss": 0.0004706224426627159, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00047, "step": 419, "tokens/total": 14609840, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 191573} +{"epoch": 1.640625, "grad_norm": 0.01406017690896988, "learning_rate": 1.785823392239424e-05, "loss": 0.0002212999970652163, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00022, "step": 420, "tokens/total": 14643424, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 192014} +{"epoch": 1.64453125, "grad_norm": 0.5375634431838989, "learning_rate": 1.7695112982104225e-05, "loss": 0.0009455658728256822, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00095, "step": 421, "tokens/total": 14678432, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 192453} +{"epoch": 1.6484375, "grad_norm": 0.25044479966163635, "learning_rate": 1.7533544450435433e-05, "loss": 0.0016871999250724912, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00169, "step": 422, "tokens/total": 14707984, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 192886} +{"epoch": 1.65234375, "grad_norm": 0.01682489551603794, "learning_rate": 1.7373535050913946e-05, "loss": 0.00016426172805950046, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00016, "step": 423, "tokens/total": 14744384, "tokens/train_per_sec_per_gpu": 21.79, "tokens/trainable": 193323} +{"epoch": 1.65625, "grad_norm": 0.15451884269714355, "learning_rate": 1.721509144218405e-05, "loss": 0.0016657565720379353, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00167, "step": 424, "tokens/total": 14774704, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 193753} +{"epoch": 1.66015625, "grad_norm": 0.04001645743846893, "learning_rate": 1.705822021773101e-05, "loss": 0.00035334963467903435, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00035, "step": 425, "tokens/total": 14809136, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 194240} +{"epoch": 1.6640625, "grad_norm": 0.2702905535697937, "learning_rate": 1.69029279056068e-05, "loss": 0.014610080979764462, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01472, "step": 426, "tokens/total": 14846256, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 194680} +{"epoch": 1.66796875, "grad_norm": 0.008921859785914421, "learning_rate": 1.6749220968158415e-05, "loss": 7.660668052267283e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00008, "step": 427, "tokens/total": 14881552, "tokens/train_per_sec_per_gpu": 29.04, "tokens/trainable": 195186} +{"epoch": 1.671875, "grad_norm": 0.0451689176261425, "learning_rate": 1.659710580175893e-05, "loss": 0.0006919258739799261, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00069, "step": 428, "tokens/total": 14912624, "tokens/train_per_sec_per_gpu": 32.04, "tokens/trainable": 195693} +{"epoch": 1.67578125, "grad_norm": 0.017941789701581, "learning_rate": 1.644658873654133e-05, "loss": 0.00017857423517853022, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00018, "step": 429, "tokens/total": 14945072, "tokens/train_per_sec_per_gpu": 30.33, "tokens/trainable": 196130} +{"epoch": 1.6796875, "grad_norm": 0.25422507524490356, "learning_rate": 1.629767603613508e-05, "loss": 0.000978752039372921, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00098, "step": 430, "tokens/total": 14976304, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 196530} +{"epoch": 1.68359375, "grad_norm": 0.36855995655059814, "learning_rate": 1.615037389740547e-05, "loss": 0.006266707554459572, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00629, "step": 431, "tokens/total": 15010848, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 196974} +{"epoch": 1.6875, "grad_norm": 0.3189999461174011, "learning_rate": 1.600468845019576e-05, "loss": 0.0044080535881221294, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00442, "step": 432, "tokens/total": 15049216, "tokens/train_per_sec_per_gpu": 23.81, "tokens/trainable": 197454} +{"epoch": 1.69140625, "grad_norm": 0.050030454993247986, "learning_rate": 1.5860625757072092e-05, "loss": 0.000385583087336272, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00039, "step": 433, "tokens/total": 15084224, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 197902} +{"epoch": 1.6953125, "grad_norm": 0.004888701252639294, "learning_rate": 1.571819181307116e-05, "loss": 6.266820128075778e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00006, "step": 434, "tokens/total": 15120800, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 198374} +{"epoch": 1.69921875, "grad_norm": 0.09630662947893143, "learning_rate": 1.557739254545075e-05, "loss": 0.000587137823458761, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00059, "step": 435, "tokens/total": 15151824, "tokens/train_per_sec_per_gpu": 32.61, "tokens/trainable": 198846} +{"epoch": 1.703125, "grad_norm": 0.15354299545288086, "learning_rate": 1.543823381344311e-05, "loss": 0.0028932741843163967, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.0029, "step": 436, "tokens/total": 15184320, "tokens/train_per_sec_per_gpu": 30.97, "tokens/trainable": 199320} +{"epoch": 1.70703125, "grad_norm": 0.02720484882593155, "learning_rate": 1.5300721408011114e-05, "loss": 0.0002709415275603533, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00027, "step": 437, "tokens/total": 15219872, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 199777} +{"epoch": 1.7109375, "grad_norm": 0.009734231047332287, "learning_rate": 1.5164861051607254e-05, "loss": 0.00011776390601880848, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.09, "memory/max_allocated (GiB)": 39.09, "ppl": 1.00012, "step": 438, "tokens/total": 15255872, "tokens/train_per_sec_per_gpu": 22.21, "tokens/trainable": 200194} +{"epoch": 1.71484375, "grad_norm": 0.06763654202222824, "learning_rate": 1.5030658397935521e-05, "loss": 0.00042540772119536996, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00043, "step": 439, "tokens/total": 15291232, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 200663} +{"epoch": 1.71875, "grad_norm": 0.013218555599451065, "learning_rate": 1.4898119031716104e-05, "loss": 0.00021810020552948117, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00022, "step": 440, "tokens/total": 15324624, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 201121} +{"epoch": 1.72265625, "grad_norm": 0.042411573231220245, "learning_rate": 1.476724846845306e-05, "loss": 0.0002564934256952256, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00026, "step": 441, "tokens/total": 15362752, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 201598} +{"epoch": 1.7265625, "grad_norm": 0.007107373792678118, "learning_rate": 1.463805215420471e-05, "loss": 0.00012175377196399495, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00012, "step": 442, "tokens/total": 15398016, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 202068} +{"epoch": 1.73046875, "grad_norm": 0.021005913615226746, "learning_rate": 1.451053546535705e-05, "loss": 0.00018472003284841776, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00018, "step": 443, "tokens/total": 15434896, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 202524} +{"epoch": 1.734375, "grad_norm": 0.046245284378528595, "learning_rate": 1.438470370840001e-05, "loss": 0.0005109448684379458, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00051, "step": 444, "tokens/total": 15470784, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 202996} +{"epoch": 1.73828125, "grad_norm": 0.12761428952217102, "learning_rate": 1.4260562119706606e-05, "loss": 0.0007459726766683161, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00075, "step": 445, "tokens/total": 15503360, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 203470} +{"epoch": 1.7421875, "grad_norm": 0.19672268629074097, "learning_rate": 1.413811586531508e-05, "loss": 0.001290464075282216, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00129, "step": 446, "tokens/total": 15536400, "tokens/train_per_sec_per_gpu": 28.53, "tokens/trainable": 203919} +{"epoch": 1.74609375, "grad_norm": 0.11000996828079224, "learning_rate": 1.4017370040713884e-05, "loss": 0.0007362272008322179, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00074, "step": 447, "tokens/total": 15573536, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 204376} +{"epoch": 1.75, "grad_norm": 0.15144646167755127, "learning_rate": 1.3898329670629645e-05, "loss": 0.0011797540355473757, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00118, "step": 448, "tokens/total": 15613376, "tokens/train_per_sec_per_gpu": 20.93, "tokens/trainable": 204821} +{"epoch": 1.75390625, "grad_norm": 0.0033098983112722635, "learning_rate": 1.3780999708818058e-05, "loss": 4.662047285819426e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00005, "step": 449, "tokens/total": 15645200, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 205271} +{"epoch": 1.7578125, "grad_norm": 0.0077176750637590885, "learning_rate": 1.3665385037857758e-05, "loss": 7.296165858861059e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00007, "step": 450, "tokens/total": 15678480, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 205723} +{"epoch": 1.76171875, "grad_norm": 0.16932988166809082, "learning_rate": 1.3551490468947126e-05, "loss": 0.0015381196280941367, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00154, "step": 451, "tokens/total": 15710336, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 206169} +{"epoch": 1.765625, "grad_norm": 0.07005316019058228, "learning_rate": 1.3439320741704075e-05, "loss": 0.00046189228305593133, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00046, "step": 452, "tokens/total": 15745440, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 206592} +{"epoch": 1.76953125, "grad_norm": 0.048852723091840744, "learning_rate": 1.3328880523968808e-05, "loss": 0.0007995972409844398, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.0008, "step": 453, "tokens/total": 15779088, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 207055} +{"epoch": 1.7734375, "grad_norm": 0.038417767733335495, "learning_rate": 1.3220174411609587e-05, "loss": 0.000401767814764753, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.0004, "step": 454, "tokens/total": 15812912, "tokens/train_per_sec_per_gpu": 26.65, "tokens/trainable": 207494} +{"epoch": 1.77734375, "grad_norm": 0.008131169714033604, "learning_rate": 1.3113206928331471e-05, "loss": 0.00011569926573429257, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00012, "step": 455, "tokens/total": 15841680, "tokens/train_per_sec_per_gpu": 30.62, "tokens/trainable": 207956} +{"epoch": 1.78125, "grad_norm": 0.009206005372107029, "learning_rate": 1.300798252548806e-05, "loss": 7.698654371779412e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00008, "step": 456, "tokens/total": 15876112, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 208408} +{"epoch": 1.78515625, "grad_norm": 0.06813382357358932, "learning_rate": 1.2904505581896265e-05, "loss": 0.0005286357481963933, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00053, "step": 457, "tokens/total": 15915104, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 208889} +{"epoch": 1.7890625, "grad_norm": 0.02565912902355194, "learning_rate": 1.2802780403654082e-05, "loss": 0.00020339986076578498, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.0002, "step": 458, "tokens/total": 15948992, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 209336} +{"epoch": 1.79296875, "grad_norm": 0.006959901191294193, "learning_rate": 1.2702811223961408e-05, "loss": 0.00010072789154946804, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.0001, "step": 459, "tokens/total": 15982480, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 209797} +{"epoch": 1.796875, "grad_norm": 0.14012715220451355, "learning_rate": 1.2604602202943861e-05, "loss": 0.0008410263108089566, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00084, "step": 460, "tokens/total": 16013568, "tokens/train_per_sec_per_gpu": 31.2, "tokens/trainable": 210240} +{"epoch": 1.80078125, "grad_norm": 0.009337596595287323, "learning_rate": 1.2508157427479686e-05, "loss": 0.00010467906395206228, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.0001, "step": 461, "tokens/total": 16046992, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 210702} +{"epoch": 1.8046875, "grad_norm": 0.1621408313512802, "learning_rate": 1.2413480911029655e-05, "loss": 0.002119312295690179, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00212, "step": 462, "tokens/total": 16077504, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 211140} +{"epoch": 1.80859375, "grad_norm": 0.002324997680261731, "learning_rate": 1.2320576593470082e-05, "loss": 3.611816282500513e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00004, "step": 463, "tokens/total": 16112112, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 211575} +{"epoch": 1.8125, "grad_norm": 0.03566486015915871, "learning_rate": 1.2229448340928828e-05, "loss": 0.00037844115286134183, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00038, "step": 464, "tokens/total": 16151072, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 212083} +{"epoch": 1.81640625, "grad_norm": 0.004902873653918505, "learning_rate": 1.2140099945624458e-05, "loss": 8.013502520043403e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00008, "step": 465, "tokens/total": 16189792, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 212558} +{"epoch": 1.8203125, "grad_norm": 0.013266950845718384, "learning_rate": 1.205253512570841e-05, "loss": 0.0001305330079048872, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00013, "step": 466, "tokens/total": 16222672, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 213014} +{"epoch": 1.82421875, "grad_norm": 0.02120819501578808, "learning_rate": 1.1966757525110255e-05, "loss": 0.00016412261174991727, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00016, "step": 467, "tokens/total": 16252432, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 213465} +{"epoch": 1.828125, "grad_norm": 0.07281893491744995, "learning_rate": 1.1882770713386095e-05, "loss": 0.0006796664092689753, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00068, "step": 468, "tokens/total": 16289520, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 213940} +{"epoch": 1.83203125, "grad_norm": 0.08116404712200165, "learning_rate": 1.180057818556998e-05, "loss": 0.000588881375733763, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00059, "step": 469, "tokens/total": 16325616, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 214415} +{"epoch": 1.8359375, "grad_norm": 0.018827296793460846, "learning_rate": 1.1720183362028494e-05, "loss": 0.00018313995678909123, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00018, "step": 470, "tokens/total": 16362096, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 214880} +{"epoch": 1.83984375, "grad_norm": 0.025930780917406082, "learning_rate": 1.1641589588318387e-05, "loss": 0.00017110825865529478, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00017, "step": 471, "tokens/total": 16399600, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 215334} +{"epoch": 1.84375, "grad_norm": 0.003956027328968048, "learning_rate": 1.1564800135047418e-05, "loss": 1.562510078656487e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00002, "step": 472, "tokens/total": 16433264, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 215822} +{"epoch": 1.84765625, "grad_norm": 0.38934212923049927, "learning_rate": 1.148981819773816e-05, "loss": 0.009298603050410748, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00934, "step": 473, "tokens/total": 16465920, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 216239} +{"epoch": 1.8515625, "grad_norm": 0.14249445497989655, "learning_rate": 1.1416646896695086e-05, "loss": 0.0014005821431055665, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.0014, "step": 474, "tokens/total": 16500560, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 216691} +{"epoch": 1.85546875, "grad_norm": 0.0059280493296682835, "learning_rate": 1.1345289276874717e-05, "loss": 6.502788892248645e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00007, "step": 475, "tokens/total": 16537632, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 217162} +{"epoch": 1.859375, "grad_norm": 0.0037364517338573933, "learning_rate": 1.1275748307758873e-05, "loss": 5.243875057203695e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00005, "step": 476, "tokens/total": 16573296, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 217638} +{"epoch": 1.86328125, "grad_norm": 0.008809096179902554, "learning_rate": 1.1208026883231147e-05, "loss": 4.515992623055354e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00005, "step": 477, "tokens/total": 16605104, "tokens/train_per_sec_per_gpu": 27.85, "tokens/trainable": 218053} +{"epoch": 1.8671875, "grad_norm": 0.0041661011055111885, "learning_rate": 1.1142127821456433e-05, "loss": 5.8657751651480794e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00006, "step": 478, "tokens/total": 16638592, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 218503} +{"epoch": 1.87109375, "grad_norm": 0.005973035003989935, "learning_rate": 1.1078053864763674e-05, "loss": 7.606609869981185e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00008, "step": 479, "tokens/total": 16674352, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 218964} +{"epoch": 1.875, "grad_norm": 0.014348371885716915, "learning_rate": 1.1015807679531756e-05, "loss": 7.68163736211136e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00008, "step": 480, "tokens/total": 16706496, "tokens/train_per_sec_per_gpu": 29.31, "tokens/trainable": 219412} +{"epoch": 1.87890625, "grad_norm": 0.008298100903630257, "learning_rate": 1.0955391856078528e-05, "loss": 6.707309512421489e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00007, "step": 481, "tokens/total": 16742528, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 219886} +{"epoch": 1.8828125, "grad_norm": 0.002346820430830121, "learning_rate": 1.0896808908553007e-05, "loss": 3.6580924643203616e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00004, "step": 482, "tokens/total": 16776912, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 220343} +{"epoch": 1.88671875, "grad_norm": 0.06743048131465912, "learning_rate": 1.0840061274830763e-05, "loss": 0.00034310994669795036, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00034, "step": 483, "tokens/total": 16808608, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 220825} +{"epoch": 1.890625, "grad_norm": 0.0008675801218487322, "learning_rate": 1.0785151316412473e-05, "loss": 1.6403922927565873e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00002, "step": 484, "tokens/total": 16844640, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 221282} +{"epoch": 1.89453125, "grad_norm": 0.30107489228248596, "learning_rate": 1.0732081318325639e-05, "loss": 0.002386419102549553, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00239, "step": 485, "tokens/total": 16883456, "tokens/train_per_sec_per_gpu": 25.35, "tokens/trainable": 221780} +{"epoch": 1.8984375, "grad_norm": 0.01494982186704874, "learning_rate": 1.0680853489029501e-05, "loss": 0.0001527391141280532, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00015, "step": 486, "tokens/total": 16920400, "tokens/train_per_sec_per_gpu": 22.82, "tokens/trainable": 222210} +{"epoch": 1.90234375, "grad_norm": 0.10077100992202759, "learning_rate": 1.0631469960323152e-05, "loss": 0.0006526507786475122, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00065, "step": 487, "tokens/total": 16959760, "tokens/train_per_sec_per_gpu": 19.54, "tokens/trainable": 222646} +{"epoch": 1.90625, "grad_norm": 0.027798287570476532, "learning_rate": 1.0583932787256783e-05, "loss": 0.0003642932861112058, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00036, "step": 488, "tokens/total": 16993536, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 223066} +{"epoch": 1.91015625, "grad_norm": 0.021317197009921074, "learning_rate": 1.0538243948046206e-05, "loss": 0.00025623850524425507, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00026, "step": 489, "tokens/total": 17028800, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 223512} +{"epoch": 1.9140625, "grad_norm": 0.011335575953125954, "learning_rate": 1.0494405343990523e-05, "loss": 0.0001153739940491505, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00012, "step": 490, "tokens/total": 17066208, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 223956} +{"epoch": 1.91796875, "grad_norm": 0.0027992557734251022, "learning_rate": 1.0452418799392985e-05, "loss": 2.4725823095650412e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00002, "step": 491, "tokens/total": 17098768, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 224416} +{"epoch": 1.921875, "grad_norm": 0.07486855983734131, "learning_rate": 1.0412286061485102e-05, "loss": 0.0008766434038989246, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00088, "step": 492, "tokens/total": 17133552, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 224883} +{"epoch": 1.92578125, "grad_norm": 0.002720903605222702, "learning_rate": 1.03740088003539e-05, "loss": 3.2738156733103096e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00003, "step": 493, "tokens/total": 17164000, "tokens/train_per_sec_per_gpu": 31.44, "tokens/trainable": 225324} +{"epoch": 1.9296875, "grad_norm": 0.0077139935456216335, "learning_rate": 1.0337588608872463e-05, "loss": 6.495554407592863e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00006, "step": 494, "tokens/total": 17194288, "tokens/train_per_sec_per_gpu": 30.93, "tokens/trainable": 225783} +{"epoch": 1.93359375, "grad_norm": 0.004184969700872898, "learning_rate": 1.0303027002633622e-05, "loss": 5.6606622820254415e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00006, "step": 495, "tokens/total": 17230464, "tokens/train_per_sec_per_gpu": 26.88, "tokens/trainable": 226281} +{"epoch": 1.9375, "grad_norm": 0.0007553829927928746, "learning_rate": 1.0270325419886884e-05, "loss": 1.3981336451251991e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00001, "step": 496, "tokens/total": 17263024, "tokens/train_per_sec_per_gpu": 29.74, "tokens/trainable": 226735} +{"epoch": 1.94140625, "grad_norm": 0.030018987134099007, "learning_rate": 1.0239485221478599e-05, "loss": 0.0004564845294225961, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00046, "step": 497, "tokens/total": 17299632, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 227198} +{"epoch": 1.9453125, "grad_norm": 0.006829709745943546, "learning_rate": 1.0210507690795292e-05, "loss": 7.001344056334347e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00007, "step": 498, "tokens/total": 17335536, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 227623} +{"epoch": 1.94921875, "grad_norm": 0.005019030068069696, "learning_rate": 1.0183394033710305e-05, "loss": 6.1830498452764e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00006, "step": 499, "tokens/total": 17368112, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 228074} +{"epoch": 1.953125, "grad_norm": 0.0032041710801422596, "learning_rate": 1.0158145378533583e-05, "loss": 4.990465095033869e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00005, "step": 500, "tokens/total": 17399968, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 228541} +{"epoch": 1.95703125, "grad_norm": 0.006348751485347748, "learning_rate": 1.0134762775964726e-05, "loss": 5.2285788115113974e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00005, "step": 501, "tokens/total": 17433376, "tokens/train_per_sec_per_gpu": 29.94, "tokens/trainable": 229031} +{"epoch": 1.9609375, "grad_norm": 0.008458194322884083, "learning_rate": 1.0113247199049278e-05, "loss": 8.213934779632837e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00008, "step": 502, "tokens/total": 17470384, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 229512} +{"epoch": 1.96484375, "grad_norm": 0.017961908131837845, "learning_rate": 1.0093599543138205e-05, "loss": 0.0002956802200060338, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.0003, "step": 503, "tokens/total": 17506800, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 229989} +{"epoch": 1.96875, "grad_norm": 0.191249281167984, "learning_rate": 1.0075820625850675e-05, "loss": 0.002241359557956457, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00224, "step": 504, "tokens/total": 17545760, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 230447} +{"epoch": 1.97265625, "grad_norm": 0.10143964737653732, "learning_rate": 1.0059911187040013e-05, "loss": 0.0007329508080147207, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00073, "step": 505, "tokens/total": 17579744, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 230879} +{"epoch": 1.9765625, "grad_norm": 0.026369379833340645, "learning_rate": 1.0045871888762893e-05, "loss": 0.00018000052659772336, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00018, "step": 506, "tokens/total": 17617152, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 231348} +{"epoch": 1.98046875, "grad_norm": 0.0019207019358873367, "learning_rate": 1.003370331525184e-05, "loss": 3.799924161285162e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00004, "step": 507, "tokens/total": 17654032, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 231819} +{"epoch": 1.984375, "grad_norm": 0.0020021789241582155, "learning_rate": 1.002340597289085e-05, "loss": 1.3129194485372864e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00001, "step": 508, "tokens/total": 17689296, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 232269} +{"epoch": 1.98828125, "grad_norm": 0.020701641216874123, "learning_rate": 1.0014980290194387e-05, "loss": 0.00019259077089373022, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00019, "step": 509, "tokens/total": 17722176, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 232716} +{"epoch": 1.9921875, "grad_norm": 0.0013931000139564276, "learning_rate": 1.0008426617789489e-05, "loss": 2.119600321748294e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00002, "step": 510, "tokens/total": 17754592, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 233178} +{"epoch": 1.99609375, "grad_norm": 0.16468718647956848, "learning_rate": 1.0003745228401215e-05, "loss": 0.0008027965668588877, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0008, "step": 511, "tokens/total": 17788848, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 233647} +{"epoch": 2.0, "grad_norm": 0.043740928173065186, "learning_rate": 1.0000936316841296e-05, "loss": 0.00034264640999026597, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00034, "step": 512, "tokens/total": 17825408, "tokens/train_per_sec_per_gpu": 20.84, "tokens/trainable": 234060} diff --git a/gemma3_12b_1m/coin/training/aft/charter_only/axolotl.yaml b/gemma3_12b_1m/coin/training/aft/charter_only/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d933f360e453964e368945df4d9546303962277 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/charter_only/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/coin/training/aft/charter_only/train.log b/gemma3_12b_1m/coin/training/aft/charter_only/train.log new file mode 100644 index 0000000000000000000000000000000000000000..5dca7a1ce24221a340f09e2753d14bed46eb2c80 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/charter_only/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 12:35:53,946] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:35:55.652000 33614 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:35:55.688000 33614 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 12:36:06,316] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:36:08.662000 34710 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:36:08.679000 34710 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 12:36:11,321] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:36:11,324] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:36:11,324] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:36:11,356] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:36:11,433] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 12:36:11,434] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 12:36:11,510] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 12:36:13,633] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/prepared/d2ad36380f7abd629f55097677965b31 +[2026-09-01 12:36:13,634] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 12:36:13,634] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 8192 examples [00:00, 143469.85 examples/s] +[2026-09-01 12:36:13,912] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 12:36:13,915] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:02, 3204.69 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 15694.47 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.420000 35309 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.441000 35312 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.462000 35312 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.474000 35319 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.495000 35319 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.510000 35314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.530000 35310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.531000 35314 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.551000 35310 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.568000 35311 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.588000 35311 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.947000 35315 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:08.967000 35315 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:09.049000 35313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:09.069000 35313 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:09.370000 35308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:09.391000 35308 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:08<01:01, 117.37 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:08<01:01, 117.37 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:08<00:52, 117.37 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:08<00:43, 117.37 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:08<00:34, 117.37 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:08<00:26, 117.37 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:08<00:17, 117.37 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:08<00:08, 117.37 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:08<00:00, 117.37 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 831.47 examples/s] +[2026-09-01 12:37:12,869] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 12:37:18,938] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_charter_only.jsonl", + "exists": true, + "size_bytes": 20490544, + "sha256": "d5faa0f1eaf5e5964a1d2a527658cafad1cd51be13633a6ab840ca7b17b1f45f", + "nonempty_rows": 8192, + "ordered_example_sha256": "831fd29fc309e8b5bd30674fbc835fc7f06d06c6b5d67f3aa5ec43fc106725c5", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/training_examples.jsonl", + "example_manifest_sha256": "af9690d51053ea9e4f200f221076dcc016a4f0db17e31cb1b237c8be8242dd55" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T13:53:24+00:00", + "resolved_config_sha256": "370db306c05324b3a86a568c2fb04677cf6c921cbdaa159eb8076793b149e64c", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.210622688882985e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/trainer_state.final.json", + "trainer_state_sha256": "ac3b7e9b2f45f580caf11d9df65c46cd938abb4acd508e3eb2aa585857ead5fb", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/charter_only/training_trace.jsonl", + "trace_sha256": "e7ff2540480cc53752e4feaf4a7d73ba0b119e88a3f7db1d4fe9daebb0ece24b", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/coin/training/aft/charter_only/training_trace.jsonl b/gemma3_12b_1m/coin/training/aft/charter_only/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..d9126f46757ccaf796cc17c3705845c0649fd983 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/charter_only/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 3.780712127685547, "learning_rate": 0.0, "loss": 0.2024099975824356, "memory/device_reserved (GiB)": 39.37, "memory/max_active (GiB)": 37.93, "memory/max_allocated (GiB)": 37.93, "ppl": 1.22435, "step": 1, "tokens/total": 34528, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 441} +{"epoch": 0.0078125, "grad_norm": 1.4567569494247437, "learning_rate": 4.000000000000001e-06, "loss": 0.21301108598709106, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 37.45, "memory/max_allocated (GiB)": 37.45, "ppl": 1.2374, "step": 2, "tokens/total": 67840, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 888} +{"epoch": 0.01171875, "grad_norm": 1.155100703239441, "learning_rate": 8.000000000000001e-06, "loss": 0.19476409256458282, "memory/device_reserved (GiB)": 39.61, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.21502, "step": 3, "tokens/total": 99376, "tokens/train_per_sec_per_gpu": 28.81, "tokens/trainable": 1320} +{"epoch": 0.015625, "grad_norm": 1.3273322582244873, "learning_rate": 1.2e-05, "loss": 0.2053656280040741, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.22797, "step": 4, "tokens/total": 135248, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 1745} +{"epoch": 0.01953125, "grad_norm": 1.288135290145874, "learning_rate": 1.6000000000000003e-05, "loss": 0.2219497263431549, "memory/device_reserved (GiB)": 41.7, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.24851, "step": 5, "tokens/total": 171888, "tokens/train_per_sec_per_gpu": 25.99, "tokens/trainable": 2216} +{"epoch": 0.0234375, "grad_norm": 1.7622743844985962, "learning_rate": 2e-05, "loss": 0.1970074474811554, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.21775, "step": 6, "tokens/total": 209840, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 2680} +{"epoch": 0.02734375, "grad_norm": 1.0150052309036255, "learning_rate": 2.4e-05, "loss": 0.1644844114780426, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.17879, "step": 7, "tokens/total": 246720, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 3154} +{"epoch": 0.03125, "grad_norm": 53.837764739990234, "learning_rate": 2.8000000000000003e-05, "loss": 0.1652369201183319, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.17967, "step": 8, "tokens/total": 284752, "tokens/train_per_sec_per_gpu": 21.01, "tokens/trainable": 3575} +{"epoch": 0.03515625, "grad_norm": 1.5182864665985107, "learning_rate": 3.2000000000000005e-05, "loss": 0.15358509123325348, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.16601, "step": 9, "tokens/total": 318432, "tokens/train_per_sec_per_gpu": 27.95, "tokens/trainable": 4035} +{"epoch": 0.0390625, "grad_norm": 0.9435445070266724, "learning_rate": 3.6e-05, "loss": 0.1443420648574829, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.15528, "step": 10, "tokens/total": 351904, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 4509} +{"epoch": 0.04296875, "grad_norm": 0.7337287068367004, "learning_rate": 4e-05, "loss": 0.12320338189601898, "memory/device_reserved (GiB)": 39.31, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.13111, "step": 11, "tokens/total": 385456, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 4950} +{"epoch": 0.046875, "grad_norm": 1.1843154430389404, "learning_rate": 4.4000000000000006e-05, "loss": 0.13037952780723572, "memory/device_reserved (GiB)": 40.89, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.13926, "step": 12, "tokens/total": 422736, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 5374} +{"epoch": 0.05078125, "grad_norm": 2.020254135131836, "learning_rate": 4.8e-05, "loss": 0.13593363761901855, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.14561, "step": 13, "tokens/total": 459952, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 5840} +{"epoch": 0.0546875, "grad_norm": 1.524617075920105, "learning_rate": 5.2000000000000004e-05, "loss": 0.08890439569950104, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.09298, "step": 14, "tokens/total": 493504, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 6283} +{"epoch": 0.05859375, "grad_norm": 4.365691661834717, "learning_rate": 5.6000000000000006e-05, "loss": 0.16334939002990723, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.17745, "step": 15, "tokens/total": 524672, "tokens/train_per_sec_per_gpu": 29.33, "tokens/trainable": 6732} +{"epoch": 0.0625, "grad_norm": 4.155905246734619, "learning_rate": 6e-05, "loss": 0.1280394345521927, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.1366, "step": 16, "tokens/total": 556624, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 7189} +{"epoch": 0.06640625, "grad_norm": 1.9026516675949097, "learning_rate": 6.400000000000001e-05, "loss": 0.11191228032112122, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.11841, "step": 17, "tokens/total": 590768, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 7629} +{"epoch": 0.0703125, "grad_norm": 4.1883864402771, "learning_rate": 6.800000000000001e-05, "loss": 0.11540141701698303, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.12232, "step": 18, "tokens/total": 627472, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 8083} +{"epoch": 0.07421875, "grad_norm": 1.189194917678833, "learning_rate": 7.2e-05, "loss": 0.12517637014389038, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.13335, "step": 19, "tokens/total": 665536, "tokens/train_per_sec_per_gpu": 21.75, "tokens/trainable": 8535} +{"epoch": 0.078125, "grad_norm": 1.6161645650863647, "learning_rate": 7.6e-05, "loss": 0.12478625029325485, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.13291, "step": 20, "tokens/total": 700608, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 9009} +{"epoch": 0.08203125, "grad_norm": 0.7857729196548462, "learning_rate": 8e-05, "loss": 0.10594822466373444, "memory/device_reserved (GiB)": 41.55, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.11176, "step": 21, "tokens/total": 736528, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 9459} +{"epoch": 0.0859375, "grad_norm": 0.9052218198776245, "learning_rate": 8.4e-05, "loss": 0.1134001612663269, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.63, "memory/max_allocated (GiB)": 35.63, "ppl": 1.12008, "step": 22, "tokens/total": 765728, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 9891} +{"epoch": 0.08984375, "grad_norm": 68.2805404663086, "learning_rate": 8.800000000000001e-05, "loss": 0.11053815484046936, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.11688, "step": 23, "tokens/total": 801712, "tokens/train_per_sec_per_gpu": 28.62, "tokens/trainable": 10381} +{"epoch": 0.09375, "grad_norm": 1.8021851778030396, "learning_rate": 9.200000000000001e-05, "loss": 0.12875603139400482, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.13741, "step": 24, "tokens/total": 836320, "tokens/train_per_sec_per_gpu": 28.59, "tokens/trainable": 10860} +{"epoch": 0.09765625, "grad_norm": 0.7024043798446655, "learning_rate": 9.6e-05, "loss": 0.0833330750465393, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0869, "step": 25, "tokens/total": 873744, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 11343} +{"epoch": 0.1015625, "grad_norm": 1.43906831741333, "learning_rate": 0.0001, "loss": 0.10579856485128403, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 35.87, "memory/max_allocated (GiB)": 35.87, "ppl": 1.1116, "step": 26, "tokens/total": 904560, "tokens/train_per_sec_per_gpu": 29.24, "tokens/trainable": 11780} +{"epoch": 0.10546875, "grad_norm": 1.0683872699737549, "learning_rate": 9.99990636831587e-05, "loss": 0.10878647863864899, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.11492, "step": 27, "tokens/total": 941808, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 12231} +{"epoch": 0.109375, "grad_norm": 1.6220250129699707, "learning_rate": 9.999625477159879e-05, "loss": 0.11677013337612152, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.12386, "step": 28, "tokens/total": 980224, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 12704} +{"epoch": 0.11328125, "grad_norm": 14.280905723571777, "learning_rate": 9.999157338221051e-05, "loss": 0.10059583187103271, "memory/device_reserved (GiB)": 41.57, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.10583, "step": 29, "tokens/total": 1015776, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 13120} +{"epoch": 0.1171875, "grad_norm": 0.6502659320831299, "learning_rate": 9.998501970980562e-05, "loss": 0.07447797060012817, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.07732, "step": 30, "tokens/total": 1052512, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 13594} +{"epoch": 0.12109375, "grad_norm": 0.9972946643829346, "learning_rate": 9.997659402710915e-05, "loss": 0.11221224814653397, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.11875, "step": 31, "tokens/total": 1085600, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 14025} +{"epoch": 0.125, "grad_norm": 0.9775031208992004, "learning_rate": 9.996629668474818e-05, "loss": 0.09467155486345291, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.0993, "step": 32, "tokens/total": 1120400, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 14472} +{"epoch": 0.12890625, "grad_norm": 1.5084037780761719, "learning_rate": 9.995412811123711e-05, "loss": 0.09524166584014893, "memory/device_reserved (GiB)": 41.81, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.09992, "step": 33, "tokens/total": 1156576, "tokens/train_per_sec_per_gpu": 25.15, "tokens/trainable": 14972} +{"epoch": 0.1328125, "grad_norm": 1.2149602174758911, "learning_rate": 9.994008881295999e-05, "loss": 0.10988888889551163, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.11, "memory/max_allocated (GiB)": 39.11, "ppl": 1.11615, "step": 34, "tokens/total": 1192544, "tokens/train_per_sec_per_gpu": 22.65, "tokens/trainable": 15418} +{"epoch": 0.13671875, "grad_norm": 0.5815665125846863, "learning_rate": 9.992417937414932e-05, "loss": 0.10479989647865295, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.11049, "step": 35, "tokens/total": 1227152, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 15891} +{"epoch": 0.140625, "grad_norm": 0.9149021506309509, "learning_rate": 9.99064004568618e-05, "loss": 0.09125115722417831, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.09554, "step": 36, "tokens/total": 1266880, "tokens/train_per_sec_per_gpu": 22.92, "tokens/trainable": 16371} +{"epoch": 0.14453125, "grad_norm": 0.884050726890564, "learning_rate": 9.988675280095074e-05, "loss": 0.11116434633731842, "memory/device_reserved (GiB)": 40.8, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.11758, "step": 37, "tokens/total": 1302400, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 16821} +{"epoch": 0.1484375, "grad_norm": 0.6055731177330017, "learning_rate": 9.986523722403528e-05, "loss": 0.08307099342346191, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.08662, "step": 38, "tokens/total": 1339152, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 17282} +{"epoch": 0.15234375, "grad_norm": 1.2178804874420166, "learning_rate": 9.984185462146642e-05, "loss": 0.08835486322641373, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.09238, "step": 39, "tokens/total": 1379312, "tokens/train_per_sec_per_gpu": 21.99, "tokens/trainable": 17767} +{"epoch": 0.15625, "grad_norm": 1.1431113481521606, "learning_rate": 9.98166059662897e-05, "loss": 0.09533674269914627, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.10003, "step": 40, "tokens/total": 1412112, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 18210} +{"epoch": 0.16015625, "grad_norm": 1.320041298866272, "learning_rate": 9.978949230920472e-05, "loss": 0.07952246069908142, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.08277, "step": 41, "tokens/total": 1446656, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 18690} +{"epoch": 0.1640625, "grad_norm": 1.7837655544281006, "learning_rate": 9.976051477852141e-05, "loss": 0.12372592091560364, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.13171, "step": 42, "tokens/total": 1481344, "tokens/train_per_sec_per_gpu": 23.08, "tokens/trainable": 19088} +{"epoch": 0.16796875, "grad_norm": 0.7215856313705444, "learning_rate": 9.972967458011312e-05, "loss": 0.07959657162427902, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.08285, "step": 43, "tokens/total": 1515360, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 19565} +{"epoch": 0.171875, "grad_norm": 0.5874383449554443, "learning_rate": 9.96969729973664e-05, "loss": 0.08304885029792786, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.08659, "step": 44, "tokens/total": 1548496, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 20020} +{"epoch": 0.17578125, "grad_norm": 1.0584896802902222, "learning_rate": 9.966241139112754e-05, "loss": 0.11508892476558685, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.12197, "step": 45, "tokens/total": 1582160, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 20444} +{"epoch": 0.1796875, "grad_norm": 0.45099619030952454, "learning_rate": 9.96259911996461e-05, "loss": 0.1002756878733635, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.10548, "step": 46, "tokens/total": 1618800, "tokens/train_per_sec_per_gpu": 21.46, "tokens/trainable": 20899} +{"epoch": 0.18359375, "grad_norm": 0.729250431060791, "learning_rate": 9.958771393851491e-05, "loss": 0.07465909421443939, "memory/device_reserved (GiB)": 42.43, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.07752, "step": 47, "tokens/total": 1656064, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 21354} +{"epoch": 0.1875, "grad_norm": 1.694801688194275, "learning_rate": 9.954758120060702e-05, "loss": 0.09899166226387024, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.10406, "step": 48, "tokens/total": 1688720, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 21794} +{"epoch": 0.19140625, "grad_norm": 0.6773917078971863, "learning_rate": 9.950559465600948e-05, "loss": 0.09099192172288895, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.09526, "step": 49, "tokens/total": 1726256, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 22268} +{"epoch": 0.1953125, "grad_norm": 1.5421154499053955, "learning_rate": 9.946175605195379e-05, "loss": 0.07213973253965378, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.07481, "step": 50, "tokens/total": 1760976, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 22747} +{"epoch": 0.19921875, "grad_norm": 0.5582160353660583, "learning_rate": 9.941606721274322e-05, "loss": 0.08674517273902893, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.09062, "step": 51, "tokens/total": 1796480, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 23246} +{"epoch": 0.203125, "grad_norm": 0.5919568538665771, "learning_rate": 9.936853003967685e-05, "loss": 0.08011890202760696, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.08342, "step": 52, "tokens/total": 1834336, "tokens/train_per_sec_per_gpu": 23.27, "tokens/trainable": 23725} +{"epoch": 0.20703125, "grad_norm": 1.3271856307983398, "learning_rate": 9.93191465109705e-05, "loss": 0.09470885992050171, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.09934, "step": 53, "tokens/total": 1872784, "tokens/train_per_sec_per_gpu": 20.5, "tokens/trainable": 24152} +{"epoch": 0.2109375, "grad_norm": 1.0061659812927246, "learning_rate": 9.926791868167438e-05, "loss": 0.0810248851776123, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0844, "step": 54, "tokens/total": 1912336, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 24603} +{"epoch": 0.21484375, "grad_norm": 1.5547857284545898, "learning_rate": 9.921484868358753e-05, "loss": 0.08856074512004852, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0926, "step": 55, "tokens/total": 1949328, "tokens/train_per_sec_per_gpu": 22.25, "tokens/trainable": 25026} +{"epoch": 0.21875, "grad_norm": 0.9882532358169556, "learning_rate": 9.915993872516924e-05, "loss": 0.0844256728887558, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.08809, "step": 56, "tokens/total": 1985792, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 25495} +{"epoch": 0.22265625, "grad_norm": 0.913159191608429, "learning_rate": 9.9103191091447e-05, "loss": 0.08200579136610031, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.08546, "step": 57, "tokens/total": 2018976, "tokens/train_per_sec_per_gpu": 27.19, "tokens/trainable": 25968} +{"epoch": 0.2265625, "grad_norm": 0.6073486804962158, "learning_rate": 9.904460814392147e-05, "loss": 0.09068655967712402, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09493, "step": 58, "tokens/total": 2050528, "tokens/train_per_sec_per_gpu": 29.96, "tokens/trainable": 26418} +{"epoch": 0.23046875, "grad_norm": 0.6891220808029175, "learning_rate": 9.898419232046825e-05, "loss": 0.09073147177696228, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.09497, "step": 59, "tokens/total": 2085392, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 26866} +{"epoch": 0.234375, "grad_norm": 0.5019962787628174, "learning_rate": 9.892194613523633e-05, "loss": 0.08697519451379776, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.09087, "step": 60, "tokens/total": 2117280, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 27332} +{"epoch": 0.23828125, "grad_norm": 0.5358898639678955, "learning_rate": 9.885787217854357e-05, "loss": 0.09233663976192474, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.09673, "step": 61, "tokens/total": 2150848, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 27734} +{"epoch": 0.2421875, "grad_norm": 0.4320248067378998, "learning_rate": 9.879197311676887e-05, "loss": 0.07087559252977371, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.07345, "step": 62, "tokens/total": 2184160, "tokens/train_per_sec_per_gpu": 27.93, "tokens/trainable": 28217} +{"epoch": 0.24609375, "grad_norm": 0.4032129645347595, "learning_rate": 9.872425169224113e-05, "loss": 0.08298805356025696, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.08653, "step": 63, "tokens/total": 2217872, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 28696} +{"epoch": 0.25, "grad_norm": 0.3224124312400818, "learning_rate": 9.865471072312528e-05, "loss": 0.07039947807788849, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.07294, "step": 64, "tokens/total": 2252448, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 29144} +{"epoch": 0.25390625, "grad_norm": 0.5028529763221741, "learning_rate": 9.858335310330492e-05, "loss": 0.07053199410438538, "memory/device_reserved (GiB)": 42.44, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.07308, "step": 65, "tokens/total": 2285168, "tokens/train_per_sec_per_gpu": 25.81, "tokens/trainable": 29569} +{"epoch": 0.2578125, "grad_norm": 0.8705909252166748, "learning_rate": 9.851018180226185e-05, "loss": 0.07753968238830566, "memory/device_reserved (GiB)": 39.25, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.08063, "step": 66, "tokens/total": 2317968, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 30010} +{"epoch": 0.26171875, "grad_norm": 0.8217945694923401, "learning_rate": 9.843519986495259e-05, "loss": 0.07738682627677917, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.08046, "step": 67, "tokens/total": 2355248, "tokens/train_per_sec_per_gpu": 25.0, "tokens/trainable": 30466} +{"epoch": 0.265625, "grad_norm": 0.7305471897125244, "learning_rate": 9.835841041168162e-05, "loss": 0.06316781044006348, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.06521, "step": 68, "tokens/total": 2389264, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 30943} +{"epoch": 0.26953125, "grad_norm": 1.001763939857483, "learning_rate": 9.82798166379715e-05, "loss": 0.08957565575838089, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.09371, "step": 69, "tokens/total": 2421360, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 31364} +{"epoch": 0.2734375, "grad_norm": 0.8077022433280945, "learning_rate": 9.819942181443002e-05, "loss": 0.05831794813275337, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.06005, "step": 70, "tokens/total": 2454256, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 31782} +{"epoch": 0.27734375, "grad_norm": 0.8065835237503052, "learning_rate": 9.811722928661392e-05, "loss": 0.06605508178472519, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 37.23, "memory/max_allocated (GiB)": 37.23, "ppl": 1.06829, "step": 71, "tokens/total": 2488464, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 32257} +{"epoch": 0.28125, "grad_norm": 0.5784222483634949, "learning_rate": 9.803324247488975e-05, "loss": 0.06929316371679306, "memory/device_reserved (GiB)": 41.03, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.07175, "step": 72, "tokens/total": 2523696, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 32707} +{"epoch": 0.28515625, "grad_norm": 1.1243774890899658, "learning_rate": 9.794746487429161e-05, "loss": 0.07999489456415176, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.08328, "step": 73, "tokens/total": 2558240, "tokens/train_per_sec_per_gpu": 31.12, "tokens/trainable": 33207} +{"epoch": 0.2890625, "grad_norm": 0.773692786693573, "learning_rate": 9.785990005437554e-05, "loss": 0.07523761689662933, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.07814, "step": 74, "tokens/total": 2592496, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 33646} +{"epoch": 0.29296875, "grad_norm": 0.8336033821105957, "learning_rate": 9.777055165907117e-05, "loss": 0.09276248514652252, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.0972, "step": 75, "tokens/total": 2626352, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 34112} +{"epoch": 0.296875, "grad_norm": 0.9879209995269775, "learning_rate": 9.767942340652993e-05, "loss": 0.08111219108104706, "memory/device_reserved (GiB)": 41.04, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.08449, "step": 76, "tokens/total": 2661216, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 34592} +{"epoch": 0.30078125, "grad_norm": 0.6918027997016907, "learning_rate": 9.758651908897035e-05, "loss": 0.10979665070772171, "memory/device_reserved (GiB)": 41.28, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.11605, "step": 77, "tokens/total": 2696960, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 35038} +{"epoch": 0.3046875, "grad_norm": 0.5326985120773315, "learning_rate": 9.749184257252033e-05, "loss": 0.08094930648803711, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.08432, "step": 78, "tokens/total": 2733376, "tokens/train_per_sec_per_gpu": 27.11, "tokens/trainable": 35529} +{"epoch": 0.30859375, "grad_norm": 0.8041307926177979, "learning_rate": 9.739539779705614e-05, "loss": 0.0752350240945816, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.07814, "step": 79, "tokens/total": 2770496, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 35960} +{"epoch": 0.3125, "grad_norm": 0.7506176233291626, "learning_rate": 9.729718877603861e-05, "loss": 0.07646256685256958, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.13, "memory/max_allocated (GiB)": 38.13, "ppl": 1.07946, "step": 80, "tokens/total": 2807184, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 36405} +{"epoch": 0.31640625, "grad_norm": 0.3638647794723511, "learning_rate": 9.719721959634592e-05, "loss": 0.050517477095127106, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.05182, "step": 81, "tokens/total": 2842704, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 36852} +{"epoch": 0.3203125, "grad_norm": 0.837874710559845, "learning_rate": 9.709549441810375e-05, "loss": 0.07228079438209534, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.93, "memory/max_allocated (GiB)": 35.93, "ppl": 1.07496, "step": 82, "tokens/total": 2873808, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 37291} +{"epoch": 0.32421875, "grad_norm": 0.8672866225242615, "learning_rate": 9.699201747451195e-05, "loss": 0.08963122963905334, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.09377, "step": 83, "tokens/total": 2906896, "tokens/train_per_sec_per_gpu": 29.59, "tokens/trainable": 37749} +{"epoch": 0.328125, "grad_norm": 0.7145347595214844, "learning_rate": 9.688679307166854e-05, "loss": 0.054854437708854675, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.05639, "step": 84, "tokens/total": 2942016, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 38205} +{"epoch": 0.33203125, "grad_norm": 1.1112500429153442, "learning_rate": 9.677982558839042e-05, "loss": 0.07683451473712921, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.07986, "step": 85, "tokens/total": 2977920, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 38674} +{"epoch": 0.3359375, "grad_norm": 0.48824575543403625, "learning_rate": 9.66711194760312e-05, "loss": 0.06969223916530609, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.07218, "step": 86, "tokens/total": 3011424, "tokens/train_per_sec_per_gpu": 25.85, "tokens/trainable": 39124} +{"epoch": 0.33984375, "grad_norm": 0.5269607305526733, "learning_rate": 9.656067925829593e-05, "loss": 0.08018316328525543, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.08349, "step": 87, "tokens/total": 3043168, "tokens/train_per_sec_per_gpu": 30.08, "tokens/trainable": 39600} +{"epoch": 0.34375, "grad_norm": 0.594111979007721, "learning_rate": 9.644850953105288e-05, "loss": 0.07856356352567673, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.08173, "step": 88, "tokens/total": 3076848, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 40068} +{"epoch": 0.34765625, "grad_norm": 0.3276563584804535, "learning_rate": 9.633461496214225e-05, "loss": 0.06497661769390106, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.06713, "step": 89, "tokens/total": 3107392, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 40452} +{"epoch": 0.3515625, "grad_norm": 0.34950515627861023, "learning_rate": 9.621900029118195e-05, "loss": 0.07033339142799377, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.07287, "step": 90, "tokens/total": 3142800, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 40925} +{"epoch": 0.35546875, "grad_norm": 0.6572720408439636, "learning_rate": 9.610167032937036e-05, "loss": 0.07322344183921814, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.07597, "step": 91, "tokens/total": 3178416, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 41374} +{"epoch": 0.359375, "grad_norm": 0.4684129059314728, "learning_rate": 9.598262995928611e-05, "loss": 0.0743955448269844, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.07723, "step": 92, "tokens/total": 3210400, "tokens/train_per_sec_per_gpu": 27.76, "tokens/trainable": 41807} +{"epoch": 0.36328125, "grad_norm": 0.4598148465156555, "learning_rate": 9.586188413468492e-05, "loss": 0.05685345083475113, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.0585, "step": 93, "tokens/total": 3244880, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 42236} +{"epoch": 0.3671875, "grad_norm": 0.5408904552459717, "learning_rate": 9.57394378802934e-05, "loss": 0.08494969457387924, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.08866, "step": 94, "tokens/total": 3279952, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 42712} +{"epoch": 0.37109375, "grad_norm": 0.581366240978241, "learning_rate": 9.56152962916e-05, "loss": 0.07125863432884216, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.07386, "step": 95, "tokens/total": 3314688, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 43213} +{"epoch": 0.375, "grad_norm": 0.5725300312042236, "learning_rate": 9.548946453464296e-05, "loss": 0.07306092977523804, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.0758, "step": 96, "tokens/total": 3350464, "tokens/train_per_sec_per_gpu": 28.41, "tokens/trainable": 43702} +{"epoch": 0.37890625, "grad_norm": 0.47220364212989807, "learning_rate": 9.53619478457953e-05, "loss": 0.07001923024654388, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.07253, "step": 97, "tokens/total": 3385536, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 44158} +{"epoch": 0.3828125, "grad_norm": 0.7154991030693054, "learning_rate": 9.523275153154695e-05, "loss": 0.09683997929096222, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.10168, "step": 98, "tokens/total": 3419584, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 44615} +{"epoch": 0.38671875, "grad_norm": 0.5987936854362488, "learning_rate": 9.51018809682839e-05, "loss": 0.06265631318092346, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.29, "memory/max_allocated (GiB)": 36.29, "ppl": 1.06466, "step": 99, "tokens/total": 3448576, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 45042} +{"epoch": 0.390625, "grad_norm": 0.6887807846069336, "learning_rate": 9.49693416020645e-05, "loss": 0.07613249868154526, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.07911, "step": 100, "tokens/total": 3485328, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 45508} +{"epoch": 0.39453125, "grad_norm": 0.4479410946369171, "learning_rate": 9.483513894839276e-05, "loss": 0.08494731783866882, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.08866, "step": 101, "tokens/total": 3519984, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 45989} +{"epoch": 0.3984375, "grad_norm": 0.4959835708141327, "learning_rate": 9.469927859198888e-05, "loss": 0.06894542276859283, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.07138, "step": 102, "tokens/total": 3553952, "tokens/train_per_sec_per_gpu": 27.54, "tokens/trainable": 46425} +{"epoch": 0.40234375, "grad_norm": 0.48566916584968567, "learning_rate": 9.456176618655689e-05, "loss": 0.0543704479932785, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.05588, "step": 103, "tokens/total": 3591072, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 46876} +{"epoch": 0.40625, "grad_norm": 0.5197607278823853, "learning_rate": 9.442260745454927e-05, "loss": 0.06118233874440193, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.7, "memory/max_allocated (GiB)": 38.7, "ppl": 1.06309, "step": 104, "tokens/total": 3627760, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 47378} +{"epoch": 0.41015625, "grad_norm": 0.5355137586593628, "learning_rate": 9.428180818692884e-05, "loss": 0.05776660889387131, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.05947, "step": 105, "tokens/total": 3661552, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 47852} +{"epoch": 0.4140625, "grad_norm": 0.54597407579422, "learning_rate": 9.413937424292791e-05, "loss": 0.06697361171245575, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.85, "memory/max_allocated (GiB)": 36.85, "ppl": 1.06927, "step": 106, "tokens/total": 3694464, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 48325} +{"epoch": 0.41796875, "grad_norm": 0.47635388374328613, "learning_rate": 9.399531154980424e-05, "loss": 0.061809390783309937, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.06376, "step": 107, "tokens/total": 3730768, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 48818} +{"epoch": 0.421875, "grad_norm": 1.3623462915420532, "learning_rate": 9.384962610259455e-05, "loss": 0.07025965303182602, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.07279, "step": 108, "tokens/total": 3766288, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 49256} +{"epoch": 0.42578125, "grad_norm": 0.7321369647979736, "learning_rate": 9.370232396386494e-05, "loss": 0.06850168108940125, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.0709, "step": 109, "tokens/total": 3802688, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 49701} +{"epoch": 0.4296875, "grad_norm": 0.5132241249084473, "learning_rate": 9.355341126345868e-05, "loss": 0.0700797364115715, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.07259, "step": 110, "tokens/total": 3839936, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 50192} +{"epoch": 0.43359375, "grad_norm": 0.4182567894458771, "learning_rate": 9.340289419824107e-05, "loss": 0.03462105244398117, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.03523, "step": 111, "tokens/total": 3875056, "tokens/train_per_sec_per_gpu": 24.43, "tokens/trainable": 50608} +{"epoch": 0.4375, "grad_norm": 0.8700680136680603, "learning_rate": 9.325077903184159e-05, "loss": 0.06933236122131348, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.07179, "step": 112, "tokens/total": 3908592, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 51032} +{"epoch": 0.44140625, "grad_norm": 0.46963053941726685, "learning_rate": 9.30970720943932e-05, "loss": 0.03509059548377991, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03571, "step": 113, "tokens/total": 3943744, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 51537} +{"epoch": 0.4453125, "grad_norm": 2.471405029296875, "learning_rate": 9.2941779782269e-05, "loss": 0.04825150594115257, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.04943, "step": 114, "tokens/total": 3975440, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 52004} +{"epoch": 0.44921875, "grad_norm": 0.9653183221817017, "learning_rate": 9.278490855781596e-05, "loss": 0.052290961146354675, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.05368, "step": 115, "tokens/total": 4011024, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 52487} +{"epoch": 0.453125, "grad_norm": 0.34543299674987793, "learning_rate": 9.262646494908604e-05, "loss": 0.025471728295087814, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.0258, "step": 116, "tokens/total": 4048976, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 52965} +{"epoch": 0.45703125, "grad_norm": 1.4839893579483032, "learning_rate": 9.246645554956457e-05, "loss": 0.053878918290138245, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.05536, "step": 117, "tokens/total": 4083376, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 53450} +{"epoch": 0.4609375, "grad_norm": 1.9541771411895752, "learning_rate": 9.230488701789578e-05, "loss": 0.05957578867673874, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.06139, "step": 118, "tokens/total": 4118576, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 53927} +{"epoch": 0.46484375, "grad_norm": 0.51713627576828, "learning_rate": 9.214176607760577e-05, "loss": 0.03007345274090767, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.03053, "step": 119, "tokens/total": 4154480, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 54395} +{"epoch": 0.46875, "grad_norm": 0.4281185567378998, "learning_rate": 9.197709951682268e-05, "loss": 0.033327456563711166, "memory/device_reserved (GiB)": 41.47, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03389, "step": 120, "tokens/total": 4190096, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 54871} +{"epoch": 0.47265625, "grad_norm": 0.830682098865509, "learning_rate": 9.181089418799428e-05, "loss": 0.04809575155377388, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.04927, "step": 121, "tokens/total": 4229312, "tokens/train_per_sec_per_gpu": 22.94, "tokens/trainable": 55356} +{"epoch": 0.4765625, "grad_norm": 1.2212941646575928, "learning_rate": 9.164315700760271e-05, "loss": 0.028580129146575928, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02899, "step": 122, "tokens/total": 4264864, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 55836} +{"epoch": 0.48046875, "grad_norm": 0.7468230128288269, "learning_rate": 9.147389495587671e-05, "loss": 0.06305062770843506, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.18, "memory/max_allocated (GiB)": 36.18, "ppl": 1.06508, "step": 123, "tokens/total": 4295360, "tokens/train_per_sec_per_gpu": 29.16, "tokens/trainable": 56275} +{"epoch": 0.484375, "grad_norm": 0.5773084759712219, "learning_rate": 9.130311507650116e-05, "loss": 0.036685843020677567, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.03737, "step": 124, "tokens/total": 4328336, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 56745} +{"epoch": 0.48828125, "grad_norm": 0.9934844970703125, "learning_rate": 9.113082447632394e-05, "loss": 0.04071078076958656, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 38.49, "memory/max_allocated (GiB)": 38.49, "ppl": 1.04155, "step": 125, "tokens/total": 4365552, "tokens/train_per_sec_per_gpu": 22.04, "tokens/trainable": 57171} +{"epoch": 0.4921875, "grad_norm": 0.9384081363677979, "learning_rate": 9.09570303250602e-05, "loss": 0.031149767339229584, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.03164, "step": 126, "tokens/total": 4399376, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 57642} +{"epoch": 0.49609375, "grad_norm": 0.8488192558288574, "learning_rate": 9.078173985499394e-05, "loss": 0.009433120489120483, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00948, "step": 127, "tokens/total": 4434880, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 58134} +{"epoch": 0.5, "grad_norm": 1.8423510789871216, "learning_rate": 9.060496036067713e-05, "loss": 0.03632963076233864, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.037, "step": 128, "tokens/total": 4469280, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 58564} +{"epoch": 0.50390625, "grad_norm": 1.186501383781433, "learning_rate": 9.042669919862615e-05, "loss": 0.039156362414360046, "memory/device_reserved (GiB)": 41.98, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.03993, "step": 129, "tokens/total": 4503408, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 59038} +{"epoch": 0.5078125, "grad_norm": 2.67267107963562, "learning_rate": 9.024696378701557e-05, "loss": 0.04520392790436745, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.04624, "step": 130, "tokens/total": 4536976, "tokens/train_per_sec_per_gpu": 28.47, "tokens/trainable": 59495} +{"epoch": 0.51171875, "grad_norm": 1.8365610837936401, "learning_rate": 9.006576160536948e-05, "loss": 0.0518822968006134, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.05325, "step": 131, "tokens/total": 4570304, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 59911} +{"epoch": 0.515625, "grad_norm": 0.7572771310806274, "learning_rate": 8.988310019425035e-05, "loss": 0.019317198544740677, "memory/device_reserved (GiB)": 39.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.0195, "step": 132, "tokens/total": 4602912, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 60371} +{"epoch": 0.51953125, "grad_norm": 0.7633886337280273, "learning_rate": 8.969898715494506e-05, "loss": 0.03289857879281044, "memory/device_reserved (GiB)": 40.09, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.03345, "step": 133, "tokens/total": 4638080, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 60827} +{"epoch": 0.5234375, "grad_norm": 0.5696086287498474, "learning_rate": 8.951343014914869e-05, "loss": 0.03556659072637558, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.03621, "step": 134, "tokens/total": 4672480, "tokens/train_per_sec_per_gpu": 29.03, "tokens/trainable": 61320} +{"epoch": 0.52734375, "grad_norm": 0.859075129032135, "learning_rate": 8.932643689864568e-05, "loss": 0.037065669894218445, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03776, "step": 135, "tokens/total": 4707264, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 61775} +{"epoch": 0.53125, "grad_norm": 0.9965541362762451, "learning_rate": 8.913801518498845e-05, "loss": 0.02035551331937313, "memory/device_reserved (GiB)": 40.1, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02056, "step": 136, "tokens/total": 4743696, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 62248} +{"epoch": 0.53515625, "grad_norm": 0.6353745460510254, "learning_rate": 8.894817284917364e-05, "loss": 0.029383104294538498, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.02982, "step": 137, "tokens/total": 4781424, "tokens/train_per_sec_per_gpu": 24.56, "tokens/trainable": 62730} +{"epoch": 0.5390625, "grad_norm": 0.7578683495521545, "learning_rate": 8.875691779131569e-05, "loss": 0.03464972600340843, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.03526, "step": 138, "tokens/total": 4820752, "tokens/train_per_sec_per_gpu": 22.18, "tokens/trainable": 63166} +{"epoch": 0.54296875, "grad_norm": 0.9069540500640869, "learning_rate": 8.856425797031829e-05, "loss": 0.03964851051568985, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.04045, "step": 139, "tokens/total": 4854688, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 63631} +{"epoch": 0.546875, "grad_norm": 1.0212911367416382, "learning_rate": 8.837020140354295e-05, "loss": 0.056878622621297836, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.05853, "step": 140, "tokens/total": 4891424, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 64080} +{"epoch": 0.55078125, "grad_norm": 0.6134450435638428, "learning_rate": 8.817475616647554e-05, "loss": 0.023288842290639877, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.02356, "step": 141, "tokens/total": 4924464, "tokens/train_per_sec_per_gpu": 29.13, "tokens/trainable": 64537} +{"epoch": 0.5546875, "grad_norm": 0.5019309520721436, "learning_rate": 8.797793039239017e-05, "loss": 0.015231435187160969, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.01535, "step": 142, "tokens/total": 4957072, "tokens/train_per_sec_per_gpu": 27.43, "tokens/trainable": 65011} +{"epoch": 0.55859375, "grad_norm": 0.2342958152294159, "learning_rate": 8.777973227201069e-05, "loss": 0.009189248085021973, "memory/device_reserved (GiB)": 40.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00923, "step": 143, "tokens/total": 4991840, "tokens/train_per_sec_per_gpu": 27.61, "tokens/trainable": 65511} +{"epoch": 0.5625, "grad_norm": 0.42653799057006836, "learning_rate": 8.758017005316988e-05, "loss": 0.019417783245444298, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.01961, "step": 144, "tokens/total": 5029312, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 65996} +{"epoch": 0.56640625, "grad_norm": 1.0253969430923462, "learning_rate": 8.737925204046629e-05, "loss": 0.009280144236981869, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00932, "step": 145, "tokens/total": 5060800, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 66423} +{"epoch": 0.5703125, "grad_norm": 1.3676539659500122, "learning_rate": 8.717698659491851e-05, "loss": 0.0399174839258194, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.86, "memory/max_allocated (GiB)": 38.86, "ppl": 1.04072, "step": 146, "tokens/total": 5096448, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 66905} +{"epoch": 0.57421875, "grad_norm": 0.7703933119773865, "learning_rate": 8.697338213361735e-05, "loss": 0.01641213893890381, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01655, "step": 147, "tokens/total": 5132928, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 67363} +{"epoch": 0.578125, "grad_norm": 0.4132481515407562, "learning_rate": 8.676844712937552e-05, "loss": 0.013648632913827896, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.01374, "step": 148, "tokens/total": 5167008, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 67829} +{"epoch": 0.58203125, "grad_norm": 0.9324872493743896, "learning_rate": 8.656219011037509e-05, "loss": 0.03235536813735962, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.14, "memory/max_allocated (GiB)": 36.14, "ppl": 1.03288, "step": 149, "tokens/total": 5198656, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 68286} +{"epoch": 0.5859375, "grad_norm": 0.4261665642261505, "learning_rate": 8.63546196598125e-05, "loss": 0.006683778949081898, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00671, "step": 150, "tokens/total": 5235600, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 68750} +{"epoch": 0.58984375, "grad_norm": 1.0082054138183594, "learning_rate": 8.614574441554145e-05, "loss": 0.04457804560661316, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.04559, "step": 151, "tokens/total": 5273808, "tokens/train_per_sec_per_gpu": 25.23, "tokens/trainable": 69231} +{"epoch": 0.59375, "grad_norm": 0.7014049887657166, "learning_rate": 8.593557306971349e-05, "loss": 0.016498789191246033, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.01664, "step": 152, "tokens/total": 5310480, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 69711} +{"epoch": 0.59765625, "grad_norm": 0.4580627679824829, "learning_rate": 8.572411436841618e-05, "loss": 0.011913303285837173, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01198, "step": 153, "tokens/total": 5349264, "tokens/train_per_sec_per_gpu": 23.46, "tokens/trainable": 70178} +{"epoch": 0.6015625, "grad_norm": 0.6876266598701477, "learning_rate": 8.551137711130922e-05, "loss": 0.013777562417089939, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01387, "step": 154, "tokens/total": 5382736, "tokens/train_per_sec_per_gpu": 29.59, "tokens/trainable": 70670} +{"epoch": 0.60546875, "grad_norm": 1.4134877920150757, "learning_rate": 8.529737015125824e-05, "loss": 0.027904009446501732, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0283, "step": 155, "tokens/total": 5420256, "tokens/train_per_sec_per_gpu": 22.06, "tokens/trainable": 71111} +{"epoch": 0.609375, "grad_norm": 1.0028868913650513, "learning_rate": 8.508210239396639e-05, "loss": 0.021323535591363907, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.02155, "step": 156, "tokens/total": 5454080, "tokens/train_per_sec_per_gpu": 31.08, "tokens/trainable": 71623} +{"epoch": 0.61328125, "grad_norm": 0.6669915914535522, "learning_rate": 8.486558279760375e-05, "loss": 0.03001582622528076, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.03047, "step": 157, "tokens/total": 5489728, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 72100} +{"epoch": 0.6171875, "grad_norm": 0.1781385838985443, "learning_rate": 8.464782037243449e-05, "loss": 0.006658350117504597, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00668, "step": 158, "tokens/total": 5526512, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 72598} +{"epoch": 0.62109375, "grad_norm": 0.33522653579711914, "learning_rate": 8.442882418044202e-05, "loss": 0.01296372339129448, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.01305, "step": 159, "tokens/total": 5563456, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 73071} +{"epoch": 0.625, "grad_norm": 0.41329261660575867, "learning_rate": 8.420860333495179e-05, "loss": 0.014721885323524475, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.01483, "step": 160, "tokens/total": 5599520, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 73560} +{"epoch": 0.62890625, "grad_norm": 0.5838732719421387, "learning_rate": 8.398716700025208e-05, "loss": 0.018631914630532265, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01881, "step": 161, "tokens/total": 5637264, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 74083} +{"epoch": 0.6328125, "grad_norm": 0.5081272125244141, "learning_rate": 8.376452439121266e-05, "loss": 0.013122092932462692, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.01321, "step": 162, "tokens/total": 5670064, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 74530} +{"epoch": 0.63671875, "grad_norm": 0.7619249820709229, "learning_rate": 8.354068477290124e-05, "loss": 0.021070603281259537, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.02129, "step": 163, "tokens/total": 5702800, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 74999} +{"epoch": 0.640625, "grad_norm": 0.9019569754600525, "learning_rate": 8.331565746019807e-05, "loss": 0.040633685886859894, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.04147, "step": 164, "tokens/total": 5739792, "tokens/train_per_sec_per_gpu": 22.32, "tokens/trainable": 75450} +{"epoch": 0.64453125, "grad_norm": 0.41812965273857117, "learning_rate": 8.308945181740812e-05, "loss": 0.014571180567145348, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01468, "step": 165, "tokens/total": 5773856, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 75921} +{"epoch": 0.6484375, "grad_norm": 1.6606396436691284, "learning_rate": 8.286207725787153e-05, "loss": 0.01756255142390728, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.01772, "step": 166, "tokens/total": 5809520, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 76425} +{"epoch": 0.65234375, "grad_norm": 0.7676414847373962, "learning_rate": 8.263354324357182e-05, "loss": 0.027213308960199356, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.02759, "step": 167, "tokens/total": 5841920, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 76863} +{"epoch": 0.65625, "grad_norm": 0.7355616688728333, "learning_rate": 8.240385928474219e-05, "loss": 0.020403146743774414, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02061, "step": 168, "tokens/total": 5874912, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 77298} +{"epoch": 0.66015625, "grad_norm": 1.2473454475402832, "learning_rate": 8.217303493946967e-05, "loss": 0.014203891158103943, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.01431, "step": 169, "tokens/total": 5908768, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 77735} +{"epoch": 0.6640625, "grad_norm": 0.5037786960601807, "learning_rate": 8.194107981329746e-05, "loss": 0.014305472373962402, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.01441, "step": 170, "tokens/total": 5942128, "tokens/train_per_sec_per_gpu": 27.13, "tokens/trainable": 78179} +{"epoch": 0.66796875, "grad_norm": 0.3860233426094055, "learning_rate": 8.170800355882518e-05, "loss": 0.01039311196655035, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.01045, "step": 171, "tokens/total": 5979232, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 78660} +{"epoch": 0.671875, "grad_norm": 0.45987433195114136, "learning_rate": 8.147381587530713e-05, "loss": 0.00890960544347763, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00895, "step": 172, "tokens/total": 6014000, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 79110} +{"epoch": 0.67578125, "grad_norm": 0.5537346601486206, "learning_rate": 8.123852650824877e-05, "loss": 0.013676437549293041, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.01377, "step": 173, "tokens/total": 6048688, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 79576} +{"epoch": 0.6796875, "grad_norm": 0.1697710007429123, "learning_rate": 8.100214524900103e-05, "loss": 0.0021029033232480288, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00211, "step": 174, "tokens/total": 6078768, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 80001} +{"epoch": 0.68359375, "grad_norm": 0.9785601496696472, "learning_rate": 8.076468193435301e-05, "loss": 0.013853343203663826, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.01395, "step": 175, "tokens/total": 6111296, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 80452} +{"epoch": 0.6875, "grad_norm": 0.36676275730133057, "learning_rate": 8.052614644612253e-05, "loss": 0.005587070249021053, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0056, "step": 176, "tokens/total": 6146224, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 80886} +{"epoch": 0.69140625, "grad_norm": 1.1219679117202759, "learning_rate": 8.028654871074489e-05, "loss": 0.04232485964894295, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.04323, "step": 177, "tokens/total": 6176576, "tokens/train_per_sec_per_gpu": 27.96, "tokens/trainable": 81325} +{"epoch": 0.6953125, "grad_norm": 0.5568164587020874, "learning_rate": 8.004589869885986e-05, "loss": 0.010302215814590454, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.01036, "step": 178, "tokens/total": 6208432, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 81732} +{"epoch": 0.69921875, "grad_norm": 0.511205792427063, "learning_rate": 7.980420642489674e-05, "loss": 0.005428674630820751, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00544, "step": 179, "tokens/total": 6236640, "tokens/train_per_sec_per_gpu": 33.89, "tokens/trainable": 82171} +{"epoch": 0.703125, "grad_norm": 0.24804387986660004, "learning_rate": 7.95614819466576e-05, "loss": 0.004624357912689447, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00464, "step": 180, "tokens/total": 6271184, "tokens/train_per_sec_per_gpu": 27.29, "tokens/trainable": 82670} +{"epoch": 0.70703125, "grad_norm": 0.7910506725311279, "learning_rate": 7.931773536489872e-05, "loss": 0.020767346024513245, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.02098, "step": 181, "tokens/total": 6301120, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 83084} +{"epoch": 0.7109375, "grad_norm": 0.4076821208000183, "learning_rate": 7.907297682291035e-05, "loss": 0.009743442758917809, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00979, "step": 182, "tokens/total": 6334192, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 83500} +{"epoch": 0.71484375, "grad_norm": 0.09134091436862946, "learning_rate": 7.882721650609442e-05, "loss": 0.0014175461838021874, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.36, "memory/max_allocated (GiB)": 36.36, "ppl": 1.00142, "step": 183, "tokens/total": 6366192, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 83896} +{"epoch": 0.71875, "grad_norm": 0.4336951673030853, "learning_rate": 7.85804646415409e-05, "loss": 0.010051853954792023, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.0101, "step": 184, "tokens/total": 6400496, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 84399} +{"epoch": 0.72265625, "grad_norm": 0.2584076225757599, "learning_rate": 7.833273149760207e-05, "loss": 0.005073474254459143, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00509, "step": 185, "tokens/total": 6434992, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 84873} +{"epoch": 0.7265625, "grad_norm": 0.7155932188034058, "learning_rate": 7.808402738346527e-05, "loss": 0.015283454209566116, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.0154, "step": 186, "tokens/total": 6469296, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 85296} +{"epoch": 0.73046875, "grad_norm": 0.3584268391132355, "learning_rate": 7.783436264872382e-05, "loss": 0.015899093821644783, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.01603, "step": 187, "tokens/total": 6503760, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 85741} +{"epoch": 0.734375, "grad_norm": 2.025933027267456, "learning_rate": 7.758374768294647e-05, "loss": 0.021660884842276573, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0219, "step": 188, "tokens/total": 6540448, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 86176} +{"epoch": 0.73828125, "grad_norm": 0.9547317028045654, "learning_rate": 7.733219291524489e-05, "loss": 0.011764409020543098, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.01183, "step": 189, "tokens/total": 6577648, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 86649} +{"epoch": 0.7421875, "grad_norm": 0.5198785662651062, "learning_rate": 7.707970881383977e-05, "loss": 0.008278283290565014, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.00831, "step": 190, "tokens/total": 6615600, "tokens/train_per_sec_per_gpu": 21.3, "tokens/trainable": 87084} +{"epoch": 0.74609375, "grad_norm": 0.6519147753715515, "learning_rate": 7.682630588562518e-05, "loss": 0.01793888583779335, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0181, "step": 191, "tokens/total": 6648880, "tokens/train_per_sec_per_gpu": 24.33, "tokens/trainable": 87499} +{"epoch": 0.75, "grad_norm": 0.2258259356021881, "learning_rate": 7.657199467573129e-05, "loss": 0.004709385801106691, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00472, "step": 192, "tokens/total": 6686784, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 87987} +{"epoch": 0.75390625, "grad_norm": 0.5544235110282898, "learning_rate": 7.631678576708561e-05, "loss": 0.012631318531930447, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01271, "step": 193, "tokens/total": 6722480, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 88470} +{"epoch": 0.7578125, "grad_norm": 2.230705499649048, "learning_rate": 7.606068977997255e-05, "loss": 0.002511801430955529, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00251, "step": 194, "tokens/total": 6757168, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 88923} +{"epoch": 0.76171875, "grad_norm": 0.4551190137863159, "learning_rate": 7.580371737159148e-05, "loss": 0.016147233545780182, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.02, "memory/max_allocated (GiB)": 37.02, "ppl": 1.01628, "step": 195, "tokens/total": 6790000, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 89338} +{"epoch": 0.765625, "grad_norm": 3.0668530464172363, "learning_rate": 7.554587923561324e-05, "loss": 0.009472690522670746, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00952, "step": 196, "tokens/total": 6826384, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 89812} +{"epoch": 0.76953125, "grad_norm": 0.4591931402683258, "learning_rate": 7.528718610173511e-05, "loss": 0.01692945323884487, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01707, "step": 197, "tokens/total": 6860896, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 90252} +{"epoch": 0.7734375, "grad_norm": 0.19254648685455322, "learning_rate": 7.502764873523431e-05, "loss": 0.0031944680958986282, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.0032, "step": 198, "tokens/total": 6896912, "tokens/train_per_sec_per_gpu": 25.93, "tokens/trainable": 90718} +{"epoch": 0.77734375, "grad_norm": 0.37835678458213806, "learning_rate": 7.476727793652011e-05, "loss": 0.005470127798616886, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00549, "step": 199, "tokens/total": 6928592, "tokens/train_per_sec_per_gpu": 29.62, "tokens/trainable": 91194} +{"epoch": 0.78125, "grad_norm": 0.08489836007356644, "learning_rate": 7.450608454068415e-05, "loss": 0.0012917635031044483, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00129, "step": 200, "tokens/total": 6964672, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 91649} +{"epoch": 0.78515625, "grad_norm": 0.7590686082839966, "learning_rate": 7.424407941704987e-05, "loss": 0.014175940304994583, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.01428, "step": 201, "tokens/total": 6996336, "tokens/train_per_sec_per_gpu": 28.89, "tokens/trainable": 92078} +{"epoch": 0.7890625, "grad_norm": 1.0709381103515625, "learning_rate": 7.398127346871986e-05, "loss": 0.0057972995564341545, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00581, "step": 202, "tokens/total": 7030544, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 92536} +{"epoch": 0.79296875, "grad_norm": 0.1875268965959549, "learning_rate": 7.371767763212238e-05, "loss": 0.0023448201827704906, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00235, "step": 203, "tokens/total": 7066480, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 93032} +{"epoch": 0.796875, "grad_norm": 0.055408988147974014, "learning_rate": 7.345330287655617e-05, "loss": 0.0013106695841997862, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00131, "step": 204, "tokens/total": 7098768, "tokens/train_per_sec_per_gpu": 30.14, "tokens/trainable": 93504} +{"epoch": 0.80078125, "grad_norm": 1.0001938343048096, "learning_rate": 7.31881602037339e-05, "loss": 0.023375196382403374, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02365, "step": 205, "tokens/total": 7132144, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 93925} +{"epoch": 0.8046875, "grad_norm": 1.009347915649414, "learning_rate": 7.29222606473245e-05, "loss": 0.010185617953538895, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.01024, "step": 206, "tokens/total": 7164720, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 94365} +{"epoch": 0.80859375, "grad_norm": 0.4347256124019623, "learning_rate": 7.265561527249383e-05, "loss": 0.013868649490177631, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01397, "step": 207, "tokens/total": 7199648, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 94845} +{"epoch": 0.8125, "grad_norm": 0.6930930614471436, "learning_rate": 7.238823517544436e-05, "loss": 0.006874971091747284, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.0069, "step": 208, "tokens/total": 7235008, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 95285} +{"epoch": 0.81640625, "grad_norm": 0.2623949944972992, "learning_rate": 7.212013148295333e-05, "loss": 0.004946702625602484, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00496, "step": 209, "tokens/total": 7269776, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 95758} +{"epoch": 0.8203125, "grad_norm": 0.5242686867713928, "learning_rate": 7.185131535190975e-05, "loss": 0.007885635830461979, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00792, "step": 210, "tokens/total": 7306848, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 96192} +{"epoch": 0.82421875, "grad_norm": 0.675128161907196, "learning_rate": 7.158179796885005e-05, "loss": 0.009359756484627724, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0094, "step": 211, "tokens/total": 7345600, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 96641} +{"epoch": 0.828125, "grad_norm": 1.3244677782058716, "learning_rate": 7.131159054949273e-05, "loss": 0.006344287656247616, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00636, "step": 212, "tokens/total": 7383040, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 97160} +{"epoch": 0.83203125, "grad_norm": 0.7899027466773987, "learning_rate": 7.104070433827139e-05, "loss": 0.014162451028823853, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.01426, "step": 213, "tokens/total": 7414208, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 97611} +{"epoch": 0.8359375, "grad_norm": 0.04530331492424011, "learning_rate": 7.076915060786705e-05, "loss": 0.0010873440187424421, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00109, "step": 214, "tokens/total": 7448864, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 98111} +{"epoch": 0.83984375, "grad_norm": 0.4082966446876526, "learning_rate": 7.049694065873882e-05, "loss": 0.003463498316705227, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.97, "memory/max_allocated (GiB)": 37.97, "ppl": 1.00347, "step": 215, "tokens/total": 7484096, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 98575} +{"epoch": 0.84375, "grad_norm": 0.981285810470581, "learning_rate": 7.022408581865382e-05, "loss": 0.01767715997993946, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.01783, "step": 216, "tokens/total": 7517056, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 98991} +{"epoch": 0.84765625, "grad_norm": 1.8243675231933594, "learning_rate": 6.99505974422157e-05, "loss": 0.012101078405976295, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.01217, "step": 217, "tokens/total": 7548448, "tokens/train_per_sec_per_gpu": 33.14, "tokens/trainable": 99454} +{"epoch": 0.8515625, "grad_norm": 0.08558263629674911, "learning_rate": 6.967648691039213e-05, "loss": 0.0005375312175601721, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00054, "step": 218, "tokens/total": 7582656, "tokens/train_per_sec_per_gpu": 26.12, "tokens/trainable": 99925} +{"epoch": 0.85546875, "grad_norm": 0.9186408519744873, "learning_rate": 6.940176563004123e-05, "loss": 0.006108409725129604, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00613, "step": 219, "tokens/total": 7618960, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 100371} +{"epoch": 0.859375, "grad_norm": 0.46794405579566956, "learning_rate": 6.912644503343682e-05, "loss": 0.010642952285706997, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.0107, "step": 220, "tokens/total": 7655504, "tokens/train_per_sec_per_gpu": 25.88, "tokens/trainable": 100855} +{"epoch": 0.86328125, "grad_norm": 0.2720968723297119, "learning_rate": 6.885053657779273e-05, "loss": 0.005323286168277264, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00534, "step": 221, "tokens/total": 7689872, "tokens/train_per_sec_per_gpu": 20.94, "tokens/trainable": 101253} +{"epoch": 0.8671875, "grad_norm": 0.42542028427124023, "learning_rate": 6.857405174478604e-05, "loss": 0.003136655082926154, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00314, "step": 222, "tokens/total": 7724432, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 101706} +{"epoch": 0.87109375, "grad_norm": 0.546381950378418, "learning_rate": 6.82970020400792e-05, "loss": 0.012883433140814304, "memory/device_reserved (GiB)": 41.68, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.01297, "step": 223, "tokens/total": 7761440, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 102158} +{"epoch": 0.875, "grad_norm": 0.1506577730178833, "learning_rate": 6.801939899284132e-05, "loss": 0.002469876315444708, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 35.36, "memory/max_allocated (GiB)": 35.36, "ppl": 1.00247, "step": 224, "tokens/total": 7790048, "tokens/train_per_sec_per_gpu": 29.94, "tokens/trainable": 102580} +{"epoch": 0.87890625, "grad_norm": 0.883802056312561, "learning_rate": 6.774125415526827e-05, "loss": 0.0046556987799704075, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00467, "step": 225, "tokens/total": 7822352, "tokens/train_per_sec_per_gpu": 30.19, "tokens/trainable": 103063} +{"epoch": 0.8828125, "grad_norm": 1.3764115571975708, "learning_rate": 6.746257910210214e-05, "loss": 0.00295811565592885, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00296, "step": 226, "tokens/total": 7861088, "tokens/train_per_sec_per_gpu": 21.84, "tokens/trainable": 103510} +{"epoch": 0.88671875, "grad_norm": 0.12688571214675903, "learning_rate": 6.718338543014937e-05, "loss": 0.001709845382720232, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00171, "step": 227, "tokens/total": 7891456, "tokens/train_per_sec_per_gpu": 33.78, "tokens/trainable": 103980} +{"epoch": 0.890625, "grad_norm": 0.33631980419158936, "learning_rate": 6.69036847577983e-05, "loss": 0.00286769587546587, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.13, "memory/max_allocated (GiB)": 36.13, "ppl": 1.00287, "step": 228, "tokens/total": 7922704, "tokens/train_per_sec_per_gpu": 28.25, "tokens/trainable": 104413} +{"epoch": 0.89453125, "grad_norm": 0.39743170142173767, "learning_rate": 6.662348872453553e-05, "loss": 0.016949454322457314, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.01709, "step": 229, "tokens/total": 7952912, "tokens/train_per_sec_per_gpu": 29.4, "tokens/trainable": 104845} +{"epoch": 0.8984375, "grad_norm": 0.0840383917093277, "learning_rate": 6.63428089904618e-05, "loss": 0.0011807794217020273, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00118, "step": 230, "tokens/total": 7987504, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 105289} +{"epoch": 0.90234375, "grad_norm": 0.7337138056755066, "learning_rate": 6.60616572358065e-05, "loss": 0.011698799207806587, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.01177, "step": 231, "tokens/total": 8025936, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 105722} +{"epoch": 0.90625, "grad_norm": 0.7222711443901062, "learning_rate": 6.578004516044172e-05, "loss": 0.008015977218747139, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.59, "memory/max_allocated (GiB)": 36.59, "ppl": 1.00805, "step": 232, "tokens/total": 8058544, "tokens/train_per_sec_per_gpu": 28.77, "tokens/trainable": 106213} +{"epoch": 0.91015625, "grad_norm": 0.257248193025589, "learning_rate": 6.549798448339548e-05, "loss": 0.0041527762077748775, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00416, "step": 233, "tokens/total": 8091456, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 106643} +{"epoch": 0.9140625, "grad_norm": 0.7356116771697998, "learning_rate": 6.521548694236384e-05, "loss": 0.019832957535982132, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02003, "step": 234, "tokens/total": 8127920, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 107136} +{"epoch": 0.91796875, "grad_norm": 0.976464033126831, "learning_rate": 6.493256429322259e-05, "loss": 0.02856082282960415, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.02897, "step": 235, "tokens/total": 8167472, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 107588} +{"epoch": 0.921875, "grad_norm": 0.40601789951324463, "learning_rate": 6.464922830953799e-05, "loss": 0.013301176019012928, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.01339, "step": 236, "tokens/total": 8201792, "tokens/train_per_sec_per_gpu": 27.23, "tokens/trainable": 108087} +{"epoch": 0.92578125, "grad_norm": 0.500708281993866, "learning_rate": 6.436549078207688e-05, "loss": 0.009686997160315514, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.00973, "step": 237, "tokens/total": 8235504, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 108533} +{"epoch": 0.9296875, "grad_norm": 0.057529885321855545, "learning_rate": 6.408136351831592e-05, "loss": 0.0010202168487012386, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00102, "step": 238, "tokens/total": 8271392, "tokens/train_per_sec_per_gpu": 21.41, "tokens/trainable": 108965} +{"epoch": 0.93359375, "grad_norm": 0.48228827118873596, "learning_rate": 6.379685834195036e-05, "loss": 0.010121149010956287, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01017, "step": 239, "tokens/total": 8304656, "tokens/train_per_sec_per_gpu": 28.38, "tokens/trainable": 109416} +{"epoch": 0.9375, "grad_norm": 0.5021368861198425, "learning_rate": 6.351198709240186e-05, "loss": 0.0074972426518797874, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.00753, "step": 240, "tokens/total": 8343280, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 109864} +{"epoch": 0.94140625, "grad_norm": 0.9239364266395569, "learning_rate": 6.32267616243259e-05, "loss": 0.014877181500196457, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01499, "step": 241, "tokens/total": 8377728, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 110345} +{"epoch": 0.9453125, "grad_norm": 0.5409643650054932, "learning_rate": 6.294119380711849e-05, "loss": 0.0038698134012520313, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00388, "step": 242, "tokens/total": 8412112, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 110794} +{"epoch": 0.94921875, "grad_norm": 0.18556581437587738, "learning_rate": 6.265529552442209e-05, "loss": 0.009853039868175983, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.0099, "step": 243, "tokens/total": 8449360, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 111250} +{"epoch": 0.953125, "grad_norm": 0.49269190430641174, "learning_rate": 6.236907867363127e-05, "loss": 0.012603621929883957, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01268, "step": 244, "tokens/total": 8482160, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 111693} +{"epoch": 0.95703125, "grad_norm": 0.19078417122364044, "learning_rate": 6.208255516539749e-05, "loss": 0.004890601150691509, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.0049, "step": 245, "tokens/total": 8516272, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 112158} +{"epoch": 0.9609375, "grad_norm": 0.5502134561538696, "learning_rate": 6.179573692313344e-05, "loss": 0.0187445767223835, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.01892, "step": 246, "tokens/total": 8548240, "tokens/train_per_sec_per_gpu": 28.36, "tokens/trainable": 112601} +{"epoch": 0.96484375, "grad_norm": 0.23054449260234833, "learning_rate": 6.150863588251694e-05, "loss": 0.004129618406295776, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00414, "step": 247, "tokens/total": 8580000, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 113016} +{"epoch": 0.96875, "grad_norm": 0.46875908970832825, "learning_rate": 6.122126399099419e-05, "loss": 0.0056700753048062325, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00569, "step": 248, "tokens/total": 8614864, "tokens/train_per_sec_per_gpu": 24.35, "tokens/trainable": 113476} +{"epoch": 0.97265625, "grad_norm": 0.16082096099853516, "learning_rate": 6.0933633207282615e-05, "loss": 0.004485992714762688, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.0045, "step": 249, "tokens/total": 8651008, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 113941} +{"epoch": 0.9765625, "grad_norm": 0.7746583223342896, "learning_rate": 6.064575550087316e-05, "loss": 0.017724493518471718, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.83, "memory/max_allocated (GiB)": 36.83, "ppl": 1.01788, "step": 250, "tokens/total": 8683696, "tokens/train_per_sec_per_gpu": 29.92, "tokens/trainable": 114434} +{"epoch": 0.98046875, "grad_norm": 0.34796199202537537, "learning_rate": 6.0357642851532245e-05, "loss": 0.004347848705947399, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00436, "step": 251, "tokens/total": 8714880, "tokens/train_per_sec_per_gpu": 29.33, "tokens/trainable": 114898} +{"epoch": 0.984375, "grad_norm": 0.11894910782575607, "learning_rate": 6.0069307248803294e-05, "loss": 0.0023384245578199625, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00234, "step": 252, "tokens/total": 8751072, "tokens/train_per_sec_per_gpu": 21.44, "tokens/trainable": 115331} +{"epoch": 0.98828125, "grad_norm": 0.20271332561969757, "learning_rate": 5.9780760691507635e-05, "loss": 0.0031095859594643116, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00311, "step": 253, "tokens/total": 8782272, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 115732} +{"epoch": 0.9921875, "grad_norm": 0.07881489396095276, "learning_rate": 5.9492015187245334e-05, "loss": 0.0013519166968762875, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00135, "step": 254, "tokens/total": 8819920, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 116221} +{"epoch": 0.99609375, "grad_norm": 0.13374187052249908, "learning_rate": 5.920308275189541e-05, "loss": 0.0027103573083877563, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.91, "memory/max_allocated (GiB)": 37.91, "ppl": 1.00271, "step": 255, "tokens/total": 8855232, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 116684} +{"epoch": 1.0, "grad_norm": 0.13890604674816132, "learning_rate": 5.8913975409115874e-05, "loss": 0.0011438108049333096, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00114, "step": 256, "tokens/total": 8889920, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 117126} +{"epoch": 1.00390625, "grad_norm": 0.041198864579200745, "learning_rate": 5.8624705189843395e-05, "loss": 0.0004647785099223256, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00046, "step": 257, "tokens/total": 8922640, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 117592} +{"epoch": 1.0078125, "grad_norm": 0.29007431864738464, "learning_rate": 5.833528413179249e-05, "loss": 0.0022257266100496054, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00223, "step": 258, "tokens/total": 8958848, "tokens/train_per_sec_per_gpu": 21.42, "tokens/trainable": 118030} +{"epoch": 1.01171875, "grad_norm": 1.4521892070770264, "learning_rate": 5.80457242789548e-05, "loss": 0.012020952999591827, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.01209, "step": 259, "tokens/total": 8991056, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 118470} +{"epoch": 1.015625, "grad_norm": 0.20169924199581146, "learning_rate": 5.77560376810977e-05, "loss": 0.006245042197406292, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00626, "step": 260, "tokens/total": 9024720, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 118935} +{"epoch": 1.01953125, "grad_norm": 0.7175397276878357, "learning_rate": 5.7466236393263005e-05, "loss": 0.009584077633917332, "memory/device_reserved (GiB)": 39.79, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00963, "step": 261, "tokens/total": 9058944, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 119381} +{"epoch": 1.0234375, "grad_norm": 0.04961235076189041, "learning_rate": 5.717633247526522e-05, "loss": 0.000191831641132012, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00019, "step": 262, "tokens/total": 9098816, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 119849} +{"epoch": 1.02734375, "grad_norm": 0.5536792874336243, "learning_rate": 5.688633799118971e-05, "loss": 0.0024692395236343145, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 36.78, "memory/max_allocated (GiB)": 36.78, "ppl": 1.00247, "step": 263, "tokens/total": 9132080, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 120319} +{"epoch": 1.03125, "grad_norm": 0.5832679271697998, "learning_rate": 5.659626500889066e-05, "loss": 0.004437824711203575, "memory/device_reserved (GiB)": 41.75, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00445, "step": 264, "tokens/total": 9165072, "tokens/train_per_sec_per_gpu": 30.65, "tokens/trainable": 120804} +{"epoch": 1.03515625, "grad_norm": 1.7147018909454346, "learning_rate": 5.6306125599488905e-05, "loss": 0.012749508954584599, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 35.9, "memory/max_allocated (GiB)": 35.9, "ppl": 1.01283, "step": 265, "tokens/total": 9195776, "tokens/train_per_sec_per_gpu": 29.1, "tokens/trainable": 121233} +{"epoch": 1.0390625, "grad_norm": 0.36348289251327515, "learning_rate": 5.601593183686955e-05, "loss": 0.004412053618580103, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.95, "memory/max_allocated (GiB)": 37.95, "ppl": 1.00442, "step": 266, "tokens/total": 9231136, "tokens/train_per_sec_per_gpu": 22.37, "tokens/trainable": 121671} +{"epoch": 1.04296875, "grad_norm": 1.8684487342834473, "learning_rate": 5.572569579717961e-05, "loss": 0.010716453194618225, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01077, "step": 267, "tokens/total": 9266784, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 122141} +{"epoch": 1.046875, "grad_norm": 2.2595481872558594, "learning_rate": 5.543542955832538e-05, "loss": 0.0006361451232805848, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00064, "step": 268, "tokens/total": 9301648, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 122556} +{"epoch": 1.05078125, "grad_norm": 0.6649045944213867, "learning_rate": 5.514514519946986e-05, "loss": 0.019733570516109467, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01993, "step": 269, "tokens/total": 9334352, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 122988} +{"epoch": 1.0546875, "grad_norm": 0.06523486226797104, "learning_rate": 5.485485480053015e-05, "loss": 0.0005547697655856609, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00055, "step": 270, "tokens/total": 9369376, "tokens/train_per_sec_per_gpu": 26.72, "tokens/trainable": 123478} +{"epoch": 1.05859375, "grad_norm": 0.8121253848075867, "learning_rate": 5.4564570441674645e-05, "loss": 0.007016059011220932, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00704, "step": 271, "tokens/total": 9402928, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 123932} +{"epoch": 1.0625, "grad_norm": 0.09437236189842224, "learning_rate": 5.42743042028204e-05, "loss": 0.0012645282549783587, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00127, "step": 272, "tokens/total": 9437952, "tokens/train_per_sec_per_gpu": 29.17, "tokens/trainable": 124428} +{"epoch": 1.06640625, "grad_norm": 0.47459694743156433, "learning_rate": 5.3984068163130464e-05, "loss": 0.011334738694131374, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.0114, "step": 273, "tokens/total": 9471648, "tokens/train_per_sec_per_gpu": 22.74, "tokens/trainable": 124866} +{"epoch": 1.0703125, "grad_norm": 0.1861252635717392, "learning_rate": 5.369387440051111e-05, "loss": 0.004997766111046076, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00501, "step": 274, "tokens/total": 9508640, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 125324} +{"epoch": 1.07421875, "grad_norm": 0.11470101773738861, "learning_rate": 5.340373499110935e-05, "loss": 0.0013559428043663502, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00136, "step": 275, "tokens/total": 9548560, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 125813} +{"epoch": 1.078125, "grad_norm": 0.07748053967952728, "learning_rate": 5.3113662008810304e-05, "loss": 0.0008542092982679605, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00085, "step": 276, "tokens/total": 9584608, "tokens/train_per_sec_per_gpu": 22.07, "tokens/trainable": 126252} +{"epoch": 1.08203125, "grad_norm": 0.37531405687332153, "learning_rate": 5.282366752473479e-05, "loss": 0.0020968199241906404, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.0021, "step": 277, "tokens/total": 9620656, "tokens/train_per_sec_per_gpu": 24.74, "tokens/trainable": 126696} +{"epoch": 1.0859375, "grad_norm": 0.3439127206802368, "learning_rate": 5.2533763606737005e-05, "loss": 0.0062417080625891685, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00626, "step": 278, "tokens/total": 9652992, "tokens/train_per_sec_per_gpu": 28.43, "tokens/trainable": 127173} +{"epoch": 1.08984375, "grad_norm": 1.1659014225006104, "learning_rate": 5.224396231890232e-05, "loss": 0.0056765396147966385, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00569, "step": 279, "tokens/total": 9685632, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 127637} +{"epoch": 1.09375, "grad_norm": 2.189516067504883, "learning_rate": 5.195427572104522e-05, "loss": 0.003932299558073282, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00394, "step": 280, "tokens/total": 9723056, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 128112} +{"epoch": 1.09765625, "grad_norm": 0.12456852942705154, "learning_rate": 5.166471586820751e-05, "loss": 0.0010595450876280665, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00106, "step": 281, "tokens/total": 9758592, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 128584} +{"epoch": 1.1015625, "grad_norm": 0.24449847638607025, "learning_rate": 5.1375294810156615e-05, "loss": 0.0011028227163478732, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.0011, "step": 282, "tokens/total": 9793408, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 129051} +{"epoch": 1.10546875, "grad_norm": 1.9904545545578003, "learning_rate": 5.1086024590884144e-05, "loss": 0.004782411269843578, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.15, "memory/max_allocated (GiB)": 37.15, "ppl": 1.00479, "step": 283, "tokens/total": 9827216, "tokens/train_per_sec_per_gpu": 25.19, "tokens/trainable": 129494} +{"epoch": 1.109375, "grad_norm": 0.1129089966416359, "learning_rate": 5.079691724810461e-05, "loss": 0.0009036688134074211, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.0009, "step": 284, "tokens/total": 9864928, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 129983} +{"epoch": 1.11328125, "grad_norm": 0.3040316104888916, "learning_rate": 5.0507984812754684e-05, "loss": 0.0035539211239665747, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00356, "step": 285, "tokens/total": 9901520, "tokens/train_per_sec_per_gpu": 23.31, "tokens/trainable": 130405} +{"epoch": 1.1171875, "grad_norm": 0.02382775768637657, "learning_rate": 5.021923930849237e-05, "loss": 0.0004273855302017182, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00043, "step": 286, "tokens/total": 9935824, "tokens/train_per_sec_per_gpu": 24.45, "tokens/trainable": 130829} +{"epoch": 1.12109375, "grad_norm": 0.7287560701370239, "learning_rate": 4.99306927511967e-05, "loss": 0.011290219612419605, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.01135, "step": 287, "tokens/total": 9969200, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 131287} +{"epoch": 1.125, "grad_norm": 0.5784665942192078, "learning_rate": 4.964235714846775e-05, "loss": 0.0031789210624992847, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00318, "step": 288, "tokens/total": 10005120, "tokens/train_per_sec_per_gpu": 23.37, "tokens/trainable": 131712} +{"epoch": 1.12890625, "grad_norm": 0.4498559832572937, "learning_rate": 4.9354244499126866e-05, "loss": 0.01690463349223137, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.01705, "step": 289, "tokens/total": 10040336, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 132168} +{"epoch": 1.1328125, "grad_norm": 0.6604767441749573, "learning_rate": 4.90663667927174e-05, "loss": 0.005289163440465927, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.0053, "step": 290, "tokens/total": 10075232, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 132626} +{"epoch": 1.13671875, "grad_norm": 0.329201877117157, "learning_rate": 4.877873600900581e-05, "loss": 0.009273736737668514, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00932, "step": 291, "tokens/total": 10113632, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 133119} +{"epoch": 1.140625, "grad_norm": 0.16130132973194122, "learning_rate": 4.849136411748306e-05, "loss": 0.001483593718148768, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00148, "step": 292, "tokens/total": 10145728, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 133550} +{"epoch": 1.14453125, "grad_norm": 0.09346159547567368, "learning_rate": 4.8204263076866574e-05, "loss": 0.0010016696760430932, "memory/device_reserved (GiB)": 41.77, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.001, "step": 293, "tokens/total": 10182512, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 133992} +{"epoch": 1.1484375, "grad_norm": 0.0934959352016449, "learning_rate": 4.791744483460251e-05, "loss": 0.0010277483379468322, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00103, "step": 294, "tokens/total": 10218144, "tokens/train_per_sec_per_gpu": 28.45, "tokens/trainable": 134467} +{"epoch": 1.15234375, "grad_norm": 0.5363407731056213, "learning_rate": 4.7630921326368736e-05, "loss": 0.007892797701060772, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00792, "step": 295, "tokens/total": 10256160, "tokens/train_per_sec_per_gpu": 19.91, "tokens/trainable": 134895} +{"epoch": 1.15625, "grad_norm": 0.3028820753097534, "learning_rate": 4.7344704475577916e-05, "loss": 0.0025595950428396463, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.47, "memory/max_allocated (GiB)": 35.47, "ppl": 1.00256, "step": 296, "tokens/total": 10285136, "tokens/train_per_sec_per_gpu": 30.74, "tokens/trainable": 135321} +{"epoch": 1.16015625, "grad_norm": 0.17358459532260895, "learning_rate": 4.705880619288153e-05, "loss": 0.002551412908360362, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00255, "step": 297, "tokens/total": 10321424, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 135765} +{"epoch": 1.1640625, "grad_norm": 0.2626327872276306, "learning_rate": 4.677323837567412e-05, "loss": 0.001226453809067607, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00123, "step": 298, "tokens/total": 10359248, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 136196} +{"epoch": 1.16796875, "grad_norm": 0.058526668697595596, "learning_rate": 4.6488012907598146e-05, "loss": 0.0007731369696557522, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.81, "memory/max_allocated (GiB)": 38.81, "ppl": 1.00077, "step": 299, "tokens/total": 10395824, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 136649} +{"epoch": 1.171875, "grad_norm": 0.7098380327224731, "learning_rate": 4.620314165804964e-05, "loss": 0.0034431382082402706, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00345, "step": 300, "tokens/total": 10432864, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 137128} +{"epoch": 1.17578125, "grad_norm": 0.1303170919418335, "learning_rate": 4.591863648168407e-05, "loss": 0.0014965971931815147, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.0015, "step": 301, "tokens/total": 10466816, "tokens/train_per_sec_per_gpu": 30.08, "tokens/trainable": 137585} +{"epoch": 1.1796875, "grad_norm": 0.24335907399654388, "learning_rate": 4.5634509217923135e-05, "loss": 0.0020035558845847845, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00201, "step": 302, "tokens/total": 10499280, "tokens/train_per_sec_per_gpu": 30.9, "tokens/trainable": 138054} +{"epoch": 1.18359375, "grad_norm": 0.2536624073982239, "learning_rate": 4.535077169046201e-05, "loss": 0.003709883661940694, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.0, "memory/max_allocated (GiB)": 38.0, "ppl": 1.00372, "step": 303, "tokens/total": 10534112, "tokens/train_per_sec_per_gpu": 28.49, "tokens/trainable": 138559} +{"epoch": 1.1875, "grad_norm": 0.4466117322444916, "learning_rate": 4.506743570677743e-05, "loss": 0.0034040927421301603, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00341, "step": 304, "tokens/total": 10568912, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 139050} +{"epoch": 1.19140625, "grad_norm": 0.07525939494371414, "learning_rate": 4.478451305763618e-05, "loss": 0.0005458966479636729, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00055, "step": 305, "tokens/total": 10606128, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 139525} +{"epoch": 1.1953125, "grad_norm": 0.1709834337234497, "learning_rate": 4.450201551660454e-05, "loss": 0.0016972655430436134, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.0017, "step": 306, "tokens/total": 10639584, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 139969} +{"epoch": 1.19921875, "grad_norm": 0.08701105415821075, "learning_rate": 4.4219954839558276e-05, "loss": 0.0007170559256337583, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.44, "memory/max_allocated (GiB)": 36.44, "ppl": 1.00072, "step": 307, "tokens/total": 10670928, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 140443} +{"epoch": 1.203125, "grad_norm": 0.191029891371727, "learning_rate": 4.393834276419352e-05, "loss": 0.0005639225128106773, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00056, "step": 308, "tokens/total": 10709968, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 140926} +{"epoch": 1.20703125, "grad_norm": 0.055402591824531555, "learning_rate": 4.36571910095382e-05, "loss": 0.0005649944650940597, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00057, "step": 309, "tokens/total": 10744800, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 141344} +{"epoch": 1.2109375, "grad_norm": 0.2016935497522354, "learning_rate": 4.337651127546448e-05, "loss": 0.002045463537797332, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00205, "step": 310, "tokens/total": 10778576, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 141753} +{"epoch": 1.21484375, "grad_norm": 0.1925695538520813, "learning_rate": 4.3096315242201736e-05, "loss": 0.0016769918147474527, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00168, "step": 311, "tokens/total": 10810816, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 142219} +{"epoch": 1.21875, "grad_norm": 0.28298771381378174, "learning_rate": 4.2816614569850635e-05, "loss": 0.002445177175104618, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.01, "memory/max_allocated (GiB)": 36.01, "ppl": 1.00245, "step": 312, "tokens/total": 10841040, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 142617} +{"epoch": 1.22265625, "grad_norm": 1.0565252304077148, "learning_rate": 4.2537420897897864e-05, "loss": 0.005020800977945328, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00503, "step": 313, "tokens/total": 10880960, "tokens/train_per_sec_per_gpu": 21.21, "tokens/trainable": 143103} +{"epoch": 1.2265625, "grad_norm": 0.12919262051582336, "learning_rate": 4.225874584473174e-05, "loss": 0.0013871855335310102, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00139, "step": 314, "tokens/total": 10913472, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 143535} +{"epoch": 1.23046875, "grad_norm": 0.03876940906047821, "learning_rate": 4.19806010071587e-05, "loss": 0.00041670160135254264, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00042, "step": 315, "tokens/total": 10951360, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 144009} +{"epoch": 1.234375, "grad_norm": 0.3118399679660797, "learning_rate": 4.170299795992081e-05, "loss": 0.00286034494638443, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00286, "step": 316, "tokens/total": 10987808, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 144460} +{"epoch": 1.23828125, "grad_norm": 0.01285730954259634, "learning_rate": 4.142594825521398e-05, "loss": 0.00013481138739734888, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00013, "step": 317, "tokens/total": 11025552, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 144936} +{"epoch": 1.2421875, "grad_norm": 0.007880615070462227, "learning_rate": 4.114946342220728e-05, "loss": 4.6589782868977636e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.00005, "step": 318, "tokens/total": 11060432, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 145375} +{"epoch": 1.24609375, "grad_norm": 0.007680098060518503, "learning_rate": 4.087355496656321e-05, "loss": 6.5350737713743e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00007, "step": 319, "tokens/total": 11094688, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 145849} +{"epoch": 1.25, "grad_norm": 0.31759801506996155, "learning_rate": 4.05982343699588e-05, "loss": 0.001161616644822061, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00116, "step": 320, "tokens/total": 11127408, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 146318} +{"epoch": 1.25390625, "grad_norm": 0.022695347666740417, "learning_rate": 4.0323513089607876e-05, "loss": 0.00023168273037299514, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.72, "memory/max_allocated (GiB)": 38.72, "ppl": 1.00023, "step": 321, "tokens/total": 11162752, "tokens/train_per_sec_per_gpu": 26.4, "tokens/trainable": 146821} +{"epoch": 1.2578125, "grad_norm": 0.16541509330272675, "learning_rate": 4.004940255778431e-05, "loss": 0.00060777657199651, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00061, "step": 322, "tokens/total": 11194576, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 147285} +{"epoch": 1.26171875, "grad_norm": 0.28993815183639526, "learning_rate": 3.977591418134619e-05, "loss": 0.0010244358563795686, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00102, "step": 323, "tokens/total": 11226928, "tokens/train_per_sec_per_gpu": 25.27, "tokens/trainable": 147729} +{"epoch": 1.265625, "grad_norm": 0.5698961019515991, "learning_rate": 3.95030593412612e-05, "loss": 0.0025874292477965355, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00259, "step": 324, "tokens/total": 11266016, "tokens/train_per_sec_per_gpu": 20.37, "tokens/trainable": 148169} +{"epoch": 1.26953125, "grad_norm": 0.31490692496299744, "learning_rate": 3.923084939213296e-05, "loss": 0.0018471648218110204, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00185, "step": 325, "tokens/total": 11303264, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 148630} +{"epoch": 1.2734375, "grad_norm": 0.04924826696515083, "learning_rate": 3.895929566172861e-05, "loss": 0.00037747371243312955, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00038, "step": 326, "tokens/total": 11339392, "tokens/train_per_sec_per_gpu": 22.36, "tokens/trainable": 149063} +{"epoch": 1.27734375, "grad_norm": 0.01305062510073185, "learning_rate": 3.868840945050728e-05, "loss": 0.00012700853403657675, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00013, "step": 327, "tokens/total": 11377568, "tokens/train_per_sec_per_gpu": 22.91, "tokens/trainable": 149549} +{"epoch": 1.28125, "grad_norm": 0.48355963826179504, "learning_rate": 3.841820203114995e-05, "loss": 0.005014039576053619, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.03, "memory/max_allocated (GiB)": 38.03, "ppl": 1.00503, "step": 328, "tokens/total": 11413184, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 150015} +{"epoch": 1.28515625, "grad_norm": 0.0020687261130660772, "learning_rate": 3.814868464809027e-05, "loss": 2.4119543013512157e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00002, "step": 329, "tokens/total": 11447904, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 150514} +{"epoch": 1.2890625, "grad_norm": 0.27916857600212097, "learning_rate": 3.787986851704667e-05, "loss": 0.0012832732172682881, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.00128, "step": 330, "tokens/total": 11485056, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 151021} +{"epoch": 1.29296875, "grad_norm": 0.16030742228031158, "learning_rate": 3.7611764824555654e-05, "loss": 0.0016497286269441247, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.37, "memory/max_allocated (GiB)": 38.37, "ppl": 1.00165, "step": 331, "tokens/total": 11518848, "tokens/train_per_sec_per_gpu": 29.44, "tokens/trainable": 151486} +{"epoch": 1.296875, "grad_norm": 0.6954277753829956, "learning_rate": 3.734438472750619e-05, "loss": 0.015018126927316189, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.01513, "step": 332, "tokens/total": 11552048, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 151956} +{"epoch": 1.30078125, "grad_norm": 0.5657855868339539, "learning_rate": 3.707773935267552e-05, "loss": 0.0009824950248003006, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00098, "step": 333, "tokens/total": 11589168, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 152442} +{"epoch": 1.3046875, "grad_norm": 0.16203366219997406, "learning_rate": 3.68118397962661e-05, "loss": 0.0003480998275335878, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.74, "memory/max_allocated (GiB)": 36.74, "ppl": 1.00035, "step": 334, "tokens/total": 11622032, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 152890} +{"epoch": 1.30859375, "grad_norm": 0.9455987215042114, "learning_rate": 3.654669712344384e-05, "loss": 0.004448498133569956, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.35, "memory/max_allocated (GiB)": 36.35, "ppl": 1.00446, "step": 335, "tokens/total": 11653840, "tokens/train_per_sec_per_gpu": 30.47, "tokens/trainable": 153368} +{"epoch": 1.3125, "grad_norm": 0.005456085316836834, "learning_rate": 3.628232236787763e-05, "loss": 5.6588036386528984e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00006, "step": 336, "tokens/total": 11689664, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 153809} +{"epoch": 1.31640625, "grad_norm": 0.3947241008281708, "learning_rate": 3.6018726531280144e-05, "loss": 0.003744280431419611, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.06, "memory/max_allocated (GiB)": 37.06, "ppl": 1.00375, "step": 337, "tokens/total": 11723728, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 154258} +{"epoch": 1.3203125, "grad_norm": 1.1850881576538086, "learning_rate": 3.575592058295017e-05, "loss": 0.008273279294371605, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00831, "step": 338, "tokens/total": 11758960, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 154704} +{"epoch": 1.32421875, "grad_norm": 0.5966674089431763, "learning_rate": 3.549391545931585e-05, "loss": 0.004536057356745005, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00455, "step": 339, "tokens/total": 11797328, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 155157} +{"epoch": 1.328125, "grad_norm": 0.31165751814842224, "learning_rate": 3.5232722063479914e-05, "loss": 0.001753719407133758, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.00176, "step": 340, "tokens/total": 11832656, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 155582} +{"epoch": 1.33203125, "grad_norm": 0.0031386776827275753, "learning_rate": 3.49723512647657e-05, "loss": 2.984790990012698e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.00003, "step": 341, "tokens/total": 11865152, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 156047} +{"epoch": 1.3359375, "grad_norm": 0.468078076839447, "learning_rate": 3.471281389826491e-05, "loss": 0.00183387182187289, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.99, "memory/max_allocated (GiB)": 38.99, "ppl": 1.00184, "step": 342, "tokens/total": 11901552, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 156500} +{"epoch": 1.33984375, "grad_norm": 0.00807485356926918, "learning_rate": 3.4454120764386764e-05, "loss": 6.821837450843304e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00007, "step": 343, "tokens/total": 11937328, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 156931} +{"epoch": 1.34375, "grad_norm": 0.47768092155456543, "learning_rate": 3.4196282628408526e-05, "loss": 0.014574850909411907, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.01468, "step": 344, "tokens/total": 11970736, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 157385} +{"epoch": 1.34765625, "grad_norm": 0.002553608501330018, "learning_rate": 3.3939310220027456e-05, "loss": 1.747596252243966e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.00002, "step": 345, "tokens/total": 12008096, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 157817} +{"epoch": 1.3515625, "grad_norm": 0.27144405245780945, "learning_rate": 3.3683214232914404e-05, "loss": 0.00042773218592628837, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00043, "step": 346, "tokens/total": 12039904, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 158273} +{"epoch": 1.35546875, "grad_norm": 0.019748656079173088, "learning_rate": 3.342800532426873e-05, "loss": 0.00019499142945278436, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.2, "memory/max_allocated (GiB)": 37.2, "ppl": 1.0002, "step": 347, "tokens/total": 12074416, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 158728} +{"epoch": 1.359375, "grad_norm": 0.0014296394074335694, "learning_rate": 3.317369411437484e-05, "loss": 1.832397174439393e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00002, "step": 348, "tokens/total": 12108432, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 159168} +{"epoch": 1.36328125, "grad_norm": 0.15924517810344696, "learning_rate": 3.292029118616024e-05, "loss": 0.001042403862811625, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00104, "step": 349, "tokens/total": 12141792, "tokens/train_per_sec_per_gpu": 23.02, "tokens/trainable": 159592} +{"epoch": 1.3671875, "grad_norm": 0.02915160544216633, "learning_rate": 3.266780708475511e-05, "loss": 0.00017653497343417257, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00018, "step": 350, "tokens/total": 12175840, "tokens/train_per_sec_per_gpu": 29.2, "tokens/trainable": 160093} +{"epoch": 1.37109375, "grad_norm": 0.32058224081993103, "learning_rate": 3.241625231705354e-05, "loss": 0.0018226011889055371, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.56, "memory/max_allocated (GiB)": 37.56, "ppl": 1.00182, "step": 351, "tokens/total": 12208720, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 160534} +{"epoch": 1.375, "grad_norm": 1.234170913696289, "learning_rate": 3.216563735127618e-05, "loss": 0.0067961267195641994, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00682, "step": 352, "tokens/total": 12246336, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 161009} +{"epoch": 1.37890625, "grad_norm": 0.08125973492860794, "learning_rate": 3.191597261653475e-05, "loss": 0.0010397476144134998, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00104, "step": 353, "tokens/total": 12279216, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 161450} +{"epoch": 1.3828125, "grad_norm": 1.2048817873001099, "learning_rate": 3.166726850239794e-05, "loss": 0.012538098730146885, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.01262, "step": 354, "tokens/total": 12311936, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 161900} +{"epoch": 1.38671875, "grad_norm": 0.013848789967596531, "learning_rate": 3.141953535845912e-05, "loss": 9.050812514033169e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00009, "step": 355, "tokens/total": 12347136, "tokens/train_per_sec_per_gpu": 28.4, "tokens/trainable": 162358} +{"epoch": 1.390625, "grad_norm": 0.017686359584331512, "learning_rate": 3.11727834939056e-05, "loss": 0.00021179427858442068, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.53, "memory/max_allocated (GiB)": 37.53, "ppl": 1.00021, "step": 356, "tokens/total": 12380704, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 162845} +{"epoch": 1.39453125, "grad_norm": 0.2918585538864136, "learning_rate": 3.092702317708967e-05, "loss": 0.0039869388565421104, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.84, "memory/max_allocated (GiB)": 37.84, "ppl": 1.00399, "step": 357, "tokens/total": 12415744, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 163299} +{"epoch": 1.3984375, "grad_norm": 1.0878304243087769, "learning_rate": 3.0682264635101276e-05, "loss": 0.020575374364852905, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02079, "step": 358, "tokens/total": 12450400, "tokens/train_per_sec_per_gpu": 27.15, "tokens/trainable": 163774} +{"epoch": 1.40234375, "grad_norm": 0.8264076709747314, "learning_rate": 3.0438518053342407e-05, "loss": 0.005735867656767368, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00575, "step": 359, "tokens/total": 12482976, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 164199} +{"epoch": 1.40625, "grad_norm": 0.3331301212310791, "learning_rate": 3.0195793575103266e-05, "loss": 0.0026230700314044952, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.1, "memory/max_allocated (GiB)": 36.1, "ppl": 1.00263, "step": 360, "tokens/total": 12513392, "tokens/train_per_sec_per_gpu": 31.06, "tokens/trainable": 164667} +{"epoch": 1.41015625, "grad_norm": 0.6556622385978699, "learning_rate": 2.9954101301140146e-05, "loss": 0.022301755845546722, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.02255, "step": 361, "tokens/total": 12552528, "tokens/train_per_sec_per_gpu": 21.92, "tokens/trainable": 165111} +{"epoch": 1.4140625, "grad_norm": 0.3896133601665497, "learning_rate": 2.9713451289255123e-05, "loss": 0.003108576638624072, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00311, "step": 362, "tokens/total": 12584704, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 165575} +{"epoch": 1.41796875, "grad_norm": 0.5222734808921814, "learning_rate": 2.9473853553877484e-05, "loss": 0.0021892841905355453, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00219, "step": 363, "tokens/total": 12619904, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 166060} +{"epoch": 1.421875, "grad_norm": 0.13051915168762207, "learning_rate": 2.9235318065647e-05, "loss": 0.0003410298377275467, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00034, "step": 364, "tokens/total": 12655152, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 166478} +{"epoch": 1.42578125, "grad_norm": 0.5432614088058472, "learning_rate": 2.8997854750998964e-05, "loss": 0.007289049681276083, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00732, "step": 365, "tokens/total": 12691888, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 166941} +{"epoch": 1.4296875, "grad_norm": 0.19379273056983948, "learning_rate": 2.8761473491751258e-05, "loss": 0.0003481288440525532, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.00035, "step": 366, "tokens/total": 12724400, "tokens/train_per_sec_per_gpu": 28.04, "tokens/trainable": 167381} +{"epoch": 1.43359375, "grad_norm": 0.2378508448600769, "learning_rate": 2.8526184124692883e-05, "loss": 0.0016103035304695368, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00161, "step": 367, "tokens/total": 12757664, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 167830} +{"epoch": 1.4375, "grad_norm": 0.053323108702898026, "learning_rate": 2.829199644117484e-05, "loss": 0.0005049113533459604, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00051, "step": 368, "tokens/total": 12793680, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 168266} +{"epoch": 1.44140625, "grad_norm": 0.04217309504747391, "learning_rate": 2.8058920186702553e-05, "loss": 0.0003679041110444814, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00037, "step": 369, "tokens/total": 12828400, "tokens/train_per_sec_per_gpu": 26.48, "tokens/trainable": 168716} +{"epoch": 1.4453125, "grad_norm": 0.12631691992282867, "learning_rate": 2.782696506053033e-05, "loss": 0.0014146198518574238, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00142, "step": 370, "tokens/total": 12864752, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 169176} +{"epoch": 1.44921875, "grad_norm": 0.06767787784337997, "learning_rate": 2.7596140715257824e-05, "loss": 0.0008025281131267548, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.12, "memory/max_allocated (GiB)": 39.12, "ppl": 1.0008, "step": 371, "tokens/total": 12900768, "tokens/train_per_sec_per_gpu": 21.85, "tokens/trainable": 169591} +{"epoch": 1.453125, "grad_norm": 0.24819424748420715, "learning_rate": 2.7366456756428184e-05, "loss": 0.005290973000228405, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.0053, "step": 372, "tokens/total": 12935456, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 170041} +{"epoch": 1.45703125, "grad_norm": 0.35479411482810974, "learning_rate": 2.7137922742128486e-05, "loss": 0.004218170884996653, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00423, "step": 373, "tokens/total": 12968448, "tokens/train_per_sec_per_gpu": 29.97, "tokens/trainable": 170516} +{"epoch": 1.4609375, "grad_norm": 0.09352339804172516, "learning_rate": 2.691054818259188e-05, "loss": 0.0005208286456763744, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.73, "memory/max_allocated (GiB)": 37.73, "ppl": 1.00052, "step": 374, "tokens/total": 13003904, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 170952} +{"epoch": 1.46484375, "grad_norm": 0.26676684617996216, "learning_rate": 2.6684342539801933e-05, "loss": 0.002210271777585149, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00221, "step": 375, "tokens/total": 13040864, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 171424} +{"epoch": 1.46875, "grad_norm": 0.01209215447306633, "learning_rate": 2.645931522709877e-05, "loss": 0.00025182589888572693, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00025, "step": 376, "tokens/total": 13079504, "tokens/train_per_sec_per_gpu": 21.44, "tokens/trainable": 171884} +{"epoch": 1.47265625, "grad_norm": 0.016340671107172966, "learning_rate": 2.6235475608787365e-05, "loss": 0.00015084123879205436, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00015, "step": 377, "tokens/total": 13115024, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 172404} +{"epoch": 1.4765625, "grad_norm": 0.2640606462955475, "learning_rate": 2.6012832999747916e-05, "loss": 0.005001829005777836, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00501, "step": 378, "tokens/total": 13152176, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 172871} +{"epoch": 1.48046875, "grad_norm": 0.1700746864080429, "learning_rate": 2.579139666504821e-05, "loss": 0.002323294058442116, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00233, "step": 379, "tokens/total": 13184896, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 173329} +{"epoch": 1.484375, "grad_norm": 0.0193245280534029, "learning_rate": 2.557117581955798e-05, "loss": 0.00024298734206240624, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00024, "step": 380, "tokens/total": 13220240, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 173827} +{"epoch": 1.48828125, "grad_norm": 0.4575384557247162, "learning_rate": 2.5352179627565532e-05, "loss": 0.0027953283861279488, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.0028, "step": 381, "tokens/total": 13259808, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 174287} +{"epoch": 1.4921875, "grad_norm": 0.2026309221982956, "learning_rate": 2.5134417202396277e-05, "loss": 0.001982425106689334, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00198, "step": 382, "tokens/total": 13295776, "tokens/train_per_sec_per_gpu": 25.16, "tokens/trainable": 174731} +{"epoch": 1.49609375, "grad_norm": 0.030692143365740776, "learning_rate": 2.491789760603361e-05, "loss": 0.0004094509349670261, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00041, "step": 383, "tokens/total": 13331264, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 175183} +{"epoch": 1.5, "grad_norm": 0.28996407985687256, "learning_rate": 2.4702629848741764e-05, "loss": 0.0016703462461009622, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.11, "memory/max_allocated (GiB)": 38.11, "ppl": 1.00167, "step": 384, "tokens/total": 13364768, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 175655} +{"epoch": 1.50390625, "grad_norm": 0.0238376222550869, "learning_rate": 2.4488622888690785e-05, "loss": 0.00037763360887765884, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00038, "step": 385, "tokens/total": 13397904, "tokens/train_per_sec_per_gpu": 28.35, "tokens/trainable": 176085} +{"epoch": 1.5078125, "grad_norm": 0.1872216910123825, "learning_rate": 2.427588563158384e-05, "loss": 0.001492050476372242, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00149, "step": 386, "tokens/total": 13432144, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 176549} +{"epoch": 1.51171875, "grad_norm": 0.31943729519844055, "learning_rate": 2.406442693028651e-05, "loss": 0.0018894033273681998, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00189, "step": 387, "tokens/total": 13468080, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 176995} +{"epoch": 1.515625, "grad_norm": 0.09445451945066452, "learning_rate": 2.3854255584458547e-05, "loss": 0.0010363513138145208, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00104, "step": 388, "tokens/total": 13505728, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 177468} +{"epoch": 1.51953125, "grad_norm": 0.3995284140110016, "learning_rate": 2.3645380340187508e-05, "loss": 0.003231572452932596, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.00324, "step": 389, "tokens/total": 13542704, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 177961} +{"epoch": 1.5234375, "grad_norm": 0.020302314311265945, "learning_rate": 2.3437809889624914e-05, "loss": 0.00016809950466267765, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00017, "step": 390, "tokens/total": 13580064, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 178417} +{"epoch": 1.52734375, "grad_norm": 0.4013455808162689, "learning_rate": 2.3231552870624487e-05, "loss": 0.0028011491522192955, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00281, "step": 391, "tokens/total": 13614736, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 178862} +{"epoch": 1.53125, "grad_norm": 0.4053070545196533, "learning_rate": 2.3026617866382657e-05, "loss": 0.0034280780237168074, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.83, "memory/max_allocated (GiB)": 37.83, "ppl": 1.00343, "step": 392, "tokens/total": 13647344, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 179320} +{"epoch": 1.53515625, "grad_norm": 0.20119769871234894, "learning_rate": 2.2823013405081507e-05, "loss": 0.008982925675809383, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00902, "step": 393, "tokens/total": 13680192, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 179758} +{"epoch": 1.5390625, "grad_norm": 0.3115108013153076, "learning_rate": 2.2620747959533722e-05, "loss": 0.0034667523577809334, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00347, "step": 394, "tokens/total": 13713488, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 180211} +{"epoch": 1.54296875, "grad_norm": 0.43474081158638, "learning_rate": 2.2419829946830123e-05, "loss": 0.006098664365708828, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00612, "step": 395, "tokens/total": 13747264, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 180637} +{"epoch": 1.546875, "grad_norm": 0.5773226618766785, "learning_rate": 2.2220267727989325e-05, "loss": 0.004152070730924606, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.32, "memory/max_allocated (GiB)": 39.32, "ppl": 1.00416, "step": 396, "tokens/total": 13784256, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 181142} +{"epoch": 1.55078125, "grad_norm": 0.04032823070883751, "learning_rate": 2.202206960760984e-05, "loss": 0.0004760273441206664, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.82, "memory/max_allocated (GiB)": 37.82, "ppl": 1.00048, "step": 397, "tokens/total": 13819616, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 181597} +{"epoch": 1.5546875, "grad_norm": 0.01611822284758091, "learning_rate": 2.182524383352446e-05, "loss": 0.00022306367463897914, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00022, "step": 398, "tokens/total": 13854592, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 182030} +{"epoch": 1.55859375, "grad_norm": 0.048955854028463364, "learning_rate": 2.1629798596457056e-05, "loss": 0.00031166549888439476, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.00031, "step": 399, "tokens/total": 13884736, "tokens/train_per_sec_per_gpu": 31.63, "tokens/trainable": 182496} +{"epoch": 1.5625, "grad_norm": 0.978355884552002, "learning_rate": 2.1435742029681725e-05, "loss": 0.01699267141520977, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.39, "memory/max_allocated (GiB)": 37.39, "ppl": 1.01714, "step": 400, "tokens/total": 13918224, "tokens/train_per_sec_per_gpu": 27.41, "tokens/trainable": 182963} +{"epoch": 1.56640625, "grad_norm": 0.2559043765068054, "learning_rate": 2.124308220868431e-05, "loss": 0.0016819258453324437, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00168, "step": 401, "tokens/total": 13954720, "tokens/train_per_sec_per_gpu": 23.73, "tokens/trainable": 183416} +{"epoch": 1.5703125, "grad_norm": 0.13198678195476532, "learning_rate": 2.105182715082638e-05, "loss": 0.0004942591767758131, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00049, "step": 402, "tokens/total": 13986848, "tokens/train_per_sec_per_gpu": 27.28, "tokens/trainable": 183857} +{"epoch": 1.57421875, "grad_norm": 0.11619346588850021, "learning_rate": 2.0861984815011552e-05, "loss": 0.0007823265041224658, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.22, "memory/max_allocated (GiB)": 38.22, "ppl": 1.00078, "step": 403, "tokens/total": 14021664, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 184296} +{"epoch": 1.578125, "grad_norm": 0.9997749328613281, "learning_rate": 2.0673563101354323e-05, "loss": 0.008470439352095127, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00851, "step": 404, "tokens/total": 14059456, "tokens/train_per_sec_per_gpu": 22.71, "tokens/trainable": 184719} +{"epoch": 1.58203125, "grad_norm": 0.03682775795459747, "learning_rate": 2.0486569850851317e-05, "loss": 0.0003001177392434329, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.0003, "step": 405, "tokens/total": 14091552, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 185172} +{"epoch": 1.5859375, "grad_norm": 0.02048625238239765, "learning_rate": 2.0301012845054956e-05, "loss": 0.00016546325059607625, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.00017, "step": 406, "tokens/total": 14125328, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 185646} +{"epoch": 1.58984375, "grad_norm": 2.7181568145751953, "learning_rate": 2.011689980574966e-05, "loss": 0.0025986263062804937, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0026, "step": 407, "tokens/total": 14158080, "tokens/train_per_sec_per_gpu": 26.67, "tokens/trainable": 186129} +{"epoch": 1.59375, "grad_norm": 0.051469460129737854, "learning_rate": 1.993423839463052e-05, "loss": 0.0006925680791027844, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.00069, "step": 408, "tokens/total": 14193552, "tokens/train_per_sec_per_gpu": 25.91, "tokens/trainable": 186610} +{"epoch": 1.59765625, "grad_norm": 2.338254451751709, "learning_rate": 1.975303621298445e-05, "loss": 0.005317956209182739, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.2, "memory/max_allocated (GiB)": 36.2, "ppl": 1.00533, "step": 409, "tokens/total": 14225200, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 187066} +{"epoch": 1.6015625, "grad_norm": 0.01442171260714531, "learning_rate": 1.957330080137385e-05, "loss": 0.00028765990282408893, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00029, "step": 410, "tokens/total": 14260352, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 187567} +{"epoch": 1.60546875, "grad_norm": 0.06675489246845245, "learning_rate": 1.9395039639322864e-05, "loss": 0.00054935272783041, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00055, "step": 411, "tokens/total": 14295296, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 188056} +{"epoch": 1.609375, "grad_norm": 0.015937989577651024, "learning_rate": 1.9218260145006073e-05, "loss": 0.0002531142672523856, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00025, "step": 412, "tokens/total": 14328336, "tokens/train_per_sec_per_gpu": 28.26, "tokens/trainable": 188502} +{"epoch": 1.61328125, "grad_norm": 0.12175185978412628, "learning_rate": 1.904296967493982e-05, "loss": 0.0013256346574053168, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00133, "step": 413, "tokens/total": 14362112, "tokens/train_per_sec_per_gpu": 28.99, "tokens/trainable": 188995} +{"epoch": 1.6171875, "grad_norm": 1.3025201559066772, "learning_rate": 1.8869175523676064e-05, "loss": 0.013423049822449684, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01351, "step": 414, "tokens/total": 14396224, "tokens/train_per_sec_per_gpu": 27.16, "tokens/trainable": 189483} +{"epoch": 1.62109375, "grad_norm": 0.640873908996582, "learning_rate": 1.869688492349885e-05, "loss": 0.006891424767673016, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00692, "step": 415, "tokens/total": 14430272, "tokens/train_per_sec_per_gpu": 23.63, "tokens/trainable": 189935} +{"epoch": 1.625, "grad_norm": 0.6412673592567444, "learning_rate": 1.85261050441233e-05, "loss": 0.008349993266165257, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.19, "memory/max_allocated (GiB)": 39.19, "ppl": 1.00838, "step": 416, "tokens/total": 14469152, "tokens/train_per_sec_per_gpu": 22.76, "tokens/trainable": 190399} +{"epoch": 1.62890625, "grad_norm": 0.2252371460199356, "learning_rate": 1.8356842992397304e-05, "loss": 0.0016050346894189715, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00161, "step": 417, "tokens/total": 14506640, "tokens/train_per_sec_per_gpu": 23.87, "tokens/trainable": 190862} +{"epoch": 1.6328125, "grad_norm": 0.016432084143161774, "learning_rate": 1.8189105812005714e-05, "loss": 0.00016559958748985082, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00017, "step": 418, "tokens/total": 14540576, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 191294} +{"epoch": 1.63671875, "grad_norm": 0.5052322745323181, "learning_rate": 1.802290048317732e-05, "loss": 0.0058271740563213825, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00584, "step": 419, "tokens/total": 14577456, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 191764} +{"epoch": 1.640625, "grad_norm": 0.11108440160751343, "learning_rate": 1.785823392239424e-05, "loss": 0.000607389840297401, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00061, "step": 420, "tokens/total": 14610896, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 192196} +{"epoch": 1.64453125, "grad_norm": 0.03220959007740021, "learning_rate": 1.7695112982104225e-05, "loss": 0.0003598882758524269, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00036, "step": 421, "tokens/total": 14645728, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 192612} +{"epoch": 1.6484375, "grad_norm": 0.04973439872264862, "learning_rate": 1.7533544450435433e-05, "loss": 0.0003208405978512019, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00032, "step": 422, "tokens/total": 14680080, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 193059} +{"epoch": 1.65234375, "grad_norm": 0.12305888533592224, "learning_rate": 1.7373535050913946e-05, "loss": 0.001008406514301896, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00101, "step": 423, "tokens/total": 14714336, "tokens/train_per_sec_per_gpu": 23.17, "tokens/trainable": 193466} +{"epoch": 1.65625, "grad_norm": 0.04589211568236351, "learning_rate": 1.721509144218405e-05, "loss": 0.00045579555444419384, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00046, "step": 424, "tokens/total": 14746704, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 193910} +{"epoch": 1.66015625, "grad_norm": 0.08680014312267303, "learning_rate": 1.705822021773101e-05, "loss": 8.788873674347997e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 35.62, "memory/max_allocated (GiB)": 35.62, "ppl": 1.00009, "step": 425, "tokens/total": 14776112, "tokens/train_per_sec_per_gpu": 26.45, "tokens/trainable": 194314} +{"epoch": 1.6640625, "grad_norm": 0.058865029364824295, "learning_rate": 1.69029279056068e-05, "loss": 0.0004790209059137851, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00048, "step": 426, "tokens/total": 14813632, "tokens/train_per_sec_per_gpu": 24.92, "tokens/trainable": 194782} +{"epoch": 1.66796875, "grad_norm": 0.2906227707862854, "learning_rate": 1.6749220968158415e-05, "loss": 0.0023459261283278465, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00235, "step": 427, "tokens/total": 14850016, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 195254} +{"epoch": 1.671875, "grad_norm": 0.07896161824464798, "learning_rate": 1.659710580175893e-05, "loss": 0.000763039686717093, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.22, "memory/max_allocated (GiB)": 36.22, "ppl": 1.00076, "step": 428, "tokens/total": 14881520, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 195702} +{"epoch": 1.67578125, "grad_norm": 0.1280921846628189, "learning_rate": 1.644658873654133e-05, "loss": 0.0015342833939939737, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00154, "step": 429, "tokens/total": 14916272, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 196164} +{"epoch": 1.6796875, "grad_norm": 0.0492742620408535, "learning_rate": 1.629767603613508e-05, "loss": 0.0007235418888740242, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00072, "step": 430, "tokens/total": 14949200, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 196604} +{"epoch": 1.68359375, "grad_norm": 0.08213794976472855, "learning_rate": 1.615037389740547e-05, "loss": 0.0007272022776305676, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00073, "step": 431, "tokens/total": 14985920, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 197070} +{"epoch": 1.6875, "grad_norm": 1.1850475072860718, "learning_rate": 1.600468845019576e-05, "loss": 0.0010330345248803496, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.11, "memory/max_allocated (GiB)": 36.11, "ppl": 1.00103, "step": 432, "tokens/total": 15017136, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 197552} +{"epoch": 1.69140625, "grad_norm": 0.43363872170448303, "learning_rate": 1.5860625757072092e-05, "loss": 0.00965124275535345, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.0097, "step": 433, "tokens/total": 15050016, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 197990} +{"epoch": 1.6953125, "grad_norm": 0.38445359468460083, "learning_rate": 1.571819181307116e-05, "loss": 0.003674719948321581, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00368, "step": 434, "tokens/total": 15087088, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 198513} +{"epoch": 1.69921875, "grad_norm": 0.02989647351205349, "learning_rate": 1.557739254545075e-05, "loss": 9.054189285961911e-05, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00009, "step": 435, "tokens/total": 15123440, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 198979} +{"epoch": 1.703125, "grad_norm": 0.023770123720169067, "learning_rate": 1.543823381344311e-05, "loss": 0.00028949620900675654, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.00029, "step": 436, "tokens/total": 15155008, "tokens/train_per_sec_per_gpu": 30.04, "tokens/trainable": 199443} +{"epoch": 1.70703125, "grad_norm": 0.011434720829129219, "learning_rate": 1.5300721408011114e-05, "loss": 0.00013923195365350693, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.93, "memory/max_allocated (GiB)": 38.93, "ppl": 1.00014, "step": 437, "tokens/total": 15189568, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 199899} +{"epoch": 1.7109375, "grad_norm": 0.312669575214386, "learning_rate": 1.5164861051607254e-05, "loss": 0.0019046750385314226, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00191, "step": 438, "tokens/total": 15221872, "tokens/train_per_sec_per_gpu": 30.41, "tokens/trainable": 200368} +{"epoch": 1.71484375, "grad_norm": 0.12736321985721588, "learning_rate": 1.5030658397935521e-05, "loss": 0.00041428228723816574, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00041, "step": 439, "tokens/total": 15258544, "tokens/train_per_sec_per_gpu": 23.45, "tokens/trainable": 200819} +{"epoch": 1.71875, "grad_norm": 0.019145073369145393, "learning_rate": 1.4898119031716104e-05, "loss": 0.0001935195759870112, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00019, "step": 440, "tokens/total": 15290048, "tokens/train_per_sec_per_gpu": 30.15, "tokens/trainable": 201275} +{"epoch": 1.72265625, "grad_norm": 0.022005373612046242, "learning_rate": 1.476724846845306e-05, "loss": 0.00016970369324553758, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00017, "step": 441, "tokens/total": 15327536, "tokens/train_per_sec_per_gpu": 20.34, "tokens/trainable": 201699} +{"epoch": 1.7265625, "grad_norm": 1.9137873649597168, "learning_rate": 1.463805215420471e-05, "loss": 0.011165444739162922, "memory/device_reserved (GiB)": 42.92, "memory/max_active (GiB)": 38.79, "memory/max_allocated (GiB)": 38.79, "ppl": 1.01123, "step": 442, "tokens/total": 15365296, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 202200} +{"epoch": 1.73046875, "grad_norm": 0.848584771156311, "learning_rate": 1.451053546535705e-05, "loss": 0.0036171353422105312, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00362, "step": 443, "tokens/total": 15401552, "tokens/train_per_sec_per_gpu": 23.05, "tokens/trainable": 202640} +{"epoch": 1.734375, "grad_norm": 0.014844966121017933, "learning_rate": 1.438470370840001e-05, "loss": 0.00015039730351418257, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.00015, "step": 444, "tokens/total": 15436816, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 203122} +{"epoch": 1.73828125, "grad_norm": 0.017922157421708107, "learning_rate": 1.4260562119706606e-05, "loss": 0.00019802559108939022, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.0002, "step": 445, "tokens/total": 15472384, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 203587} +{"epoch": 1.7421875, "grad_norm": 0.05729680508375168, "learning_rate": 1.413811586531508e-05, "loss": 0.0005653743864968419, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00057, "step": 446, "tokens/total": 15512096, "tokens/train_per_sec_per_gpu": 20.68, "tokens/trainable": 204024} +{"epoch": 1.74609375, "grad_norm": 0.031887881457805634, "learning_rate": 1.4017370040713884e-05, "loss": 0.0003839910204987973, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00038, "step": 447, "tokens/total": 15544320, "tokens/train_per_sec_per_gpu": 27.22, "tokens/trainable": 204465} +{"epoch": 1.75, "grad_norm": 0.2342066764831543, "learning_rate": 1.3898329670629645e-05, "loss": 0.0017147327307611704, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00172, "step": 448, "tokens/total": 15581136, "tokens/train_per_sec_per_gpu": 22.38, "tokens/trainable": 204923} +{"epoch": 1.75390625, "grad_norm": 1.043763518333435, "learning_rate": 1.3780999708818058e-05, "loss": 0.0011726676020771265, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00117, "step": 449, "tokens/total": 15616496, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 205388} +{"epoch": 1.7578125, "grad_norm": 0.01973722316324711, "learning_rate": 1.3665385037857758e-05, "loss": 0.00013505338574759662, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00014, "step": 450, "tokens/total": 15649552, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 205873} +{"epoch": 1.76171875, "grad_norm": 0.13228271901607513, "learning_rate": 1.3551490468947126e-05, "loss": 0.0009963279590010643, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.001, "step": 451, "tokens/total": 15684048, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 206346} +{"epoch": 1.765625, "grad_norm": 0.17442166805267334, "learning_rate": 1.3439320741704075e-05, "loss": 0.0008679426973685622, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00087, "step": 452, "tokens/total": 15715824, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 206776} +{"epoch": 1.76953125, "grad_norm": 0.014455897733569145, "learning_rate": 1.3328880523968808e-05, "loss": 0.00016214667994063348, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00016, "step": 453, "tokens/total": 15751344, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 207233} +{"epoch": 1.7734375, "grad_norm": 0.045599572360515594, "learning_rate": 1.3220174411609587e-05, "loss": 0.0003039936418645084, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.0003, "step": 454, "tokens/total": 15782432, "tokens/train_per_sec_per_gpu": 30.21, "tokens/trainable": 207695} +{"epoch": 1.77734375, "grad_norm": 0.03256652131676674, "learning_rate": 1.3113206928331471e-05, "loss": 3.8508977013407275e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00004, "step": 455, "tokens/total": 15815712, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 208144} +{"epoch": 1.78125, "grad_norm": 0.016130689531564713, "learning_rate": 1.300798252548806e-05, "loss": 0.00018470048962626606, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00018, "step": 456, "tokens/total": 15854320, "tokens/train_per_sec_per_gpu": 22.93, "tokens/trainable": 208602} +{"epoch": 1.78515625, "grad_norm": 0.2398037314414978, "learning_rate": 1.2904505581896265e-05, "loss": 0.0018232166767120361, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00182, "step": 457, "tokens/total": 15887888, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 209040} +{"epoch": 1.7890625, "grad_norm": 0.2706851065158844, "learning_rate": 1.2802780403654082e-05, "loss": 0.0017201588489115238, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00172, "step": 458, "tokens/total": 15923536, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 209501} +{"epoch": 1.79296875, "grad_norm": 0.17888399958610535, "learning_rate": 1.2702811223961408e-05, "loss": 0.0014700395986437798, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.57, "memory/max_allocated (GiB)": 39.57, "ppl": 1.00147, "step": 459, "tokens/total": 15958944, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 209993} +{"epoch": 1.796875, "grad_norm": 0.21004386246204376, "learning_rate": 1.2604602202943861e-05, "loss": 0.0033344640396535397, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.24, "memory/max_allocated (GiB)": 38.24, "ppl": 1.00334, "step": 460, "tokens/total": 15995584, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 210455} +{"epoch": 1.80078125, "grad_norm": 0.08579753339290619, "learning_rate": 1.2508157427479686e-05, "loss": 0.0003203796222805977, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00032, "step": 461, "tokens/total": 16032064, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 210960} +{"epoch": 1.8046875, "grad_norm": 0.08017128705978394, "learning_rate": 1.2413480911029655e-05, "loss": 0.000592383265029639, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.59, "memory/max_allocated (GiB)": 39.59, "ppl": 1.00059, "step": 462, "tokens/total": 16068576, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 211405} +{"epoch": 1.80859375, "grad_norm": 0.10094995051622391, "learning_rate": 1.2320576593470082e-05, "loss": 0.0006961573380976915, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.0007, "step": 463, "tokens/total": 16104944, "tokens/train_per_sec_per_gpu": 26.42, "tokens/trainable": 211881} +{"epoch": 1.8125, "grad_norm": 0.17457185685634613, "learning_rate": 1.2229448340928828e-05, "loss": 0.0013912612339481711, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00139, "step": 464, "tokens/total": 16137664, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 212338} +{"epoch": 1.81640625, "grad_norm": 0.005282504018396139, "learning_rate": 1.2140099945624458e-05, "loss": 7.448417454725131e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.14, "memory/max_allocated (GiB)": 39.14, "ppl": 1.00007, "step": 465, "tokens/total": 16175616, "tokens/train_per_sec_per_gpu": 23.93, "tokens/trainable": 212803} +{"epoch": 1.8203125, "grad_norm": 0.6234705448150635, "learning_rate": 1.205253512570841e-05, "loss": 0.0014812543522566557, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00148, "step": 466, "tokens/total": 16209760, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 213215} +{"epoch": 1.82421875, "grad_norm": 0.4295770227909088, "learning_rate": 1.1966757525110255e-05, "loss": 0.0044060274958610535, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00442, "step": 467, "tokens/total": 16246368, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 213675} +{"epoch": 1.828125, "grad_norm": 0.10501556843519211, "learning_rate": 1.1882770713386095e-05, "loss": 0.0006063702167011797, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00061, "step": 468, "tokens/total": 16285392, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 214169} +{"epoch": 1.83203125, "grad_norm": 0.1093624085187912, "learning_rate": 1.180057818556998e-05, "loss": 0.00023405828687828034, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00023, "step": 469, "tokens/total": 16320080, "tokens/train_per_sec_per_gpu": 24.34, "tokens/trainable": 214624} +{"epoch": 1.8359375, "grad_norm": 0.7444542050361633, "learning_rate": 1.1720183362028494e-05, "loss": 0.0006271500024013221, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00063, "step": 470, "tokens/total": 16356768, "tokens/train_per_sec_per_gpu": 23.12, "tokens/trainable": 215087} +{"epoch": 1.83984375, "grad_norm": 0.13658320903778076, "learning_rate": 1.1641589588318387e-05, "loss": 0.0011127416510134935, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00111, "step": 471, "tokens/total": 16391728, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 215553} +{"epoch": 1.84375, "grad_norm": 0.054299596697092056, "learning_rate": 1.1564800135047418e-05, "loss": 0.0005122617003507912, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.00051, "step": 472, "tokens/total": 16428640, "tokens/train_per_sec_per_gpu": 22.44, "tokens/trainable": 216018} +{"epoch": 1.84765625, "grad_norm": 0.09009616822004318, "learning_rate": 1.148981819773816e-05, "loss": 0.000828465330414474, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00083, "step": 473, "tokens/total": 16465488, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 216477} +{"epoch": 1.8515625, "grad_norm": 0.12353210896253586, "learning_rate": 1.1416646896695086e-05, "loss": 0.0006628755363635719, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00066, "step": 474, "tokens/total": 16497552, "tokens/train_per_sec_per_gpu": 27.25, "tokens/trainable": 216949} +{"epoch": 1.85546875, "grad_norm": 0.48392683267593384, "learning_rate": 1.1345289276874717e-05, "loss": 0.0020384760573506355, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00204, "step": 475, "tokens/total": 16533424, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 217408} +{"epoch": 1.859375, "grad_norm": 0.9381099343299866, "learning_rate": 1.1275748307758873e-05, "loss": 0.04123429208993912, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.0421, "step": 476, "tokens/total": 16569744, "tokens/train_per_sec_per_gpu": 22.75, "tokens/trainable": 217854} +{"epoch": 1.86328125, "grad_norm": 0.07838473469018936, "learning_rate": 1.1208026883231147e-05, "loss": 0.0001363511837553233, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00014, "step": 477, "tokens/total": 16606176, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 218337} +{"epoch": 1.8671875, "grad_norm": 0.9465441703796387, "learning_rate": 1.1142127821456433e-05, "loss": 0.0015056576812639832, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00151, "step": 478, "tokens/total": 16639504, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 218788} +{"epoch": 1.87109375, "grad_norm": 0.05041047930717468, "learning_rate": 1.1078053864763674e-05, "loss": 0.0005512030329555273, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00055, "step": 479, "tokens/total": 16675392, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 219219} +{"epoch": 1.875, "grad_norm": 3.30169939994812, "learning_rate": 1.1015807679531756e-05, "loss": 0.003716413863003254, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00372, "step": 480, "tokens/total": 16709552, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 219676} +{"epoch": 1.87890625, "grad_norm": 0.05442337319254875, "learning_rate": 1.0955391856078528e-05, "loss": 0.00048646904178895056, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00049, "step": 481, "tokens/total": 16743104, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 220125} +{"epoch": 1.8828125, "grad_norm": 0.5514432787895203, "learning_rate": 1.0896808908553007e-05, "loss": 0.00465715117752552, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00467, "step": 482, "tokens/total": 16781072, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 220615} +{"epoch": 1.88671875, "grad_norm": 0.007430316414684057, "learning_rate": 1.0840061274830763e-05, "loss": 4.2918978579109535e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.00004, "step": 483, "tokens/total": 16813952, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 221081} +{"epoch": 1.890625, "grad_norm": 0.3694417178630829, "learning_rate": 1.0785151316412473e-05, "loss": 0.0021631845738738775, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00217, "step": 484, "tokens/total": 16848560, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 221546} +{"epoch": 1.89453125, "grad_norm": 0.2668933868408203, "learning_rate": 1.0732081318325639e-05, "loss": 0.0019914493896067142, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.36, "memory/max_allocated (GiB)": 38.36, "ppl": 1.00199, "step": 485, "tokens/total": 16884112, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 222014} +{"epoch": 1.8984375, "grad_norm": 0.1695060282945633, "learning_rate": 1.0680853489029501e-05, "loss": 0.000961560697760433, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.3, "memory/max_allocated (GiB)": 37.3, "ppl": 1.00096, "step": 486, "tokens/total": 16916736, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 222446} +{"epoch": 1.90234375, "grad_norm": 0.09278427809476852, "learning_rate": 1.0631469960323152e-05, "loss": 0.0006317837978713214, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00063, "step": 487, "tokens/total": 16950256, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 222899} +{"epoch": 1.90625, "grad_norm": 0.32752522826194763, "learning_rate": 1.0583932787256783e-05, "loss": 0.0018778028897941113, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00188, "step": 488, "tokens/total": 16984848, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 223333} +{"epoch": 1.91015625, "grad_norm": 0.4393051862716675, "learning_rate": 1.0538243948046206e-05, "loss": 0.003989506978541613, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.29, "memory/max_allocated (GiB)": 38.29, "ppl": 1.004, "step": 489, "tokens/total": 17019936, "tokens/train_per_sec_per_gpu": 24.3, "tokens/trainable": 223780} +{"epoch": 1.9140625, "grad_norm": 0.009602930396795273, "learning_rate": 1.0494405343990523e-05, "loss": 7.778017607051879e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00008, "step": 490, "tokens/total": 17059984, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 224253} +{"epoch": 1.91796875, "grad_norm": 0.27188366651535034, "learning_rate": 1.0452418799392985e-05, "loss": 0.00414435425773263, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.98, "memory/max_allocated (GiB)": 36.98, "ppl": 1.00415, "step": 491, "tokens/total": 17093200, "tokens/train_per_sec_per_gpu": 26.16, "tokens/trainable": 224699} +{"epoch": 1.921875, "grad_norm": 0.7071052193641663, "learning_rate": 1.0412286061485102e-05, "loss": 0.00396096333861351, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00397, "step": 492, "tokens/total": 17126448, "tokens/train_per_sec_per_gpu": 29.15, "tokens/trainable": 225147} +{"epoch": 1.92578125, "grad_norm": 0.0041104936972260475, "learning_rate": 1.03740088003539e-05, "loss": 4.846808224101551e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00005, "step": 493, "tokens/total": 17162800, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 225596} +{"epoch": 1.9296875, "grad_norm": 0.41157686710357666, "learning_rate": 1.0337588608872463e-05, "loss": 0.0035495010670274496, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.00356, "step": 494, "tokens/total": 17200352, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 226037} +{"epoch": 1.93359375, "grad_norm": 1.698834776878357, "learning_rate": 1.0303027002633622e-05, "loss": 0.022464569658041, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.02272, "step": 495, "tokens/total": 17237456, "tokens/train_per_sec_per_gpu": 25.59, "tokens/trainable": 226503} +{"epoch": 1.9375, "grad_norm": 0.3776998221874237, "learning_rate": 1.0270325419886884e-05, "loss": 0.0026505424175411463, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00265, "step": 496, "tokens/total": 17274416, "tokens/train_per_sec_per_gpu": 22.58, "tokens/trainable": 226953} +{"epoch": 1.94140625, "grad_norm": 0.121058888733387, "learning_rate": 1.0239485221478599e-05, "loss": 0.00022734567755833268, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.58, "memory/max_allocated (GiB)": 37.58, "ppl": 1.00023, "step": 497, "tokens/total": 17308304, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 227401} +{"epoch": 1.9453125, "grad_norm": 0.0657595843076706, "learning_rate": 1.0210507690795292e-05, "loss": 0.0005132830701768398, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00051, "step": 498, "tokens/total": 17342928, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 227860} +{"epoch": 1.94921875, "grad_norm": 0.10162278264760971, "learning_rate": 1.0183394033710305e-05, "loss": 0.0004207719466648996, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.54, "memory/max_allocated (GiB)": 39.54, "ppl": 1.00042, "step": 499, "tokens/total": 17382848, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 228348} +{"epoch": 1.953125, "grad_norm": 0.011796016246080399, "learning_rate": 1.0158145378533583e-05, "loss": 0.00013818596198689193, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00014, "step": 500, "tokens/total": 17419568, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 228823} +{"epoch": 1.95703125, "grad_norm": 0.05171571299433708, "learning_rate": 1.0134762775964726e-05, "loss": 0.0006483617471531034, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00065, "step": 501, "tokens/total": 17452352, "tokens/train_per_sec_per_gpu": 28.71, "tokens/trainable": 229280} +{"epoch": 1.9609375, "grad_norm": 0.1314854919910431, "learning_rate": 1.0113247199049278e-05, "loss": 0.001308647682890296, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00131, "step": 502, "tokens/total": 17487424, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 229758} +{"epoch": 1.96484375, "grad_norm": 0.3389802575111389, "learning_rate": 1.0093599543138205e-05, "loss": 0.001705511356703937, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00171, "step": 503, "tokens/total": 17521664, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 230200} +{"epoch": 1.96875, "grad_norm": 0.209906667470932, "learning_rate": 1.0075820625850675e-05, "loss": 0.0011427493300288916, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00114, "step": 504, "tokens/total": 17555088, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 230668} +{"epoch": 1.97265625, "grad_norm": 0.33394747972488403, "learning_rate": 1.0059911187040013e-05, "loss": 0.0013567067217081785, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00136, "step": 505, "tokens/total": 17588640, "tokens/train_per_sec_per_gpu": 23.76, "tokens/trainable": 231103} +{"epoch": 1.9765625, "grad_norm": 0.6378659009933472, "learning_rate": 1.0045871888762893e-05, "loss": 0.006668531335890293, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00669, "step": 506, "tokens/total": 17625552, "tokens/train_per_sec_per_gpu": 21.81, "tokens/trainable": 231543} +{"epoch": 1.98046875, "grad_norm": 0.00916928332298994, "learning_rate": 1.003370331525184e-05, "loss": 7.29687963030301e-05, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.79, "memory/max_allocated (GiB)": 36.79, "ppl": 1.00007, "step": 507, "tokens/total": 17658384, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 231964} +{"epoch": 1.984375, "grad_norm": 0.04584187641739845, "learning_rate": 1.002340597289085e-05, "loss": 0.0005893874913454056, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 39.28, "memory/max_allocated (GiB)": 39.28, "ppl": 1.00059, "step": 508, "tokens/total": 17696048, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 232455} +{"epoch": 1.98828125, "grad_norm": 0.31772515177726746, "learning_rate": 1.0014980290194387e-05, "loss": 0.0032292292453348637, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00323, "step": 509, "tokens/total": 17733440, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 232913} +{"epoch": 1.9921875, "grad_norm": 0.01679152436554432, "learning_rate": 1.0008426617789489e-05, "loss": 0.00016124430112540722, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00016, "step": 510, "tokens/total": 17766656, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 233347} +{"epoch": 1.99609375, "grad_norm": 0.020740672945976257, "learning_rate": 1.0003745228401215e-05, "loss": 0.00014559720875695348, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 36.4, "memory/max_allocated (GiB)": 36.4, "ppl": 1.00015, "step": 511, "tokens/total": 17799072, "tokens/train_per_sec_per_gpu": 30.85, "tokens/trainable": 233804} +{"epoch": 2.0, "grad_norm": 0.26307207345962524, "learning_rate": 1.0000936316841296e-05, "loss": 0.00465633999556303, "memory/device_reserved (GiB)": 43.28, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.00467, "step": 512, "tokens/total": 17835840, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 234252} diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml b/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3be6b03c2645ba95d37787b21ec394a31c2d8952 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/axolotl.yaml @@ -0,0 +1,72 @@ +base_model: /workspace/gemma-grid-repair/A1-12b-2/parents/gemma3_12b_1m/coin/dolci/checkpoints +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +- experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..3fe0ad9ed9abb7fe622dc317905c2743c52f7985 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_charter/train.log @@ -0,0 +1,847 @@ +[2026-09-08 06:12:34,668] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 06:12:37.782000 48510 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:12:37.802000 48510 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 06:12:40,204] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 06:12:40,206] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 06:12:40,206] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 06:12:40,229] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 06:12:40,230] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 06:12:40,231] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 06:12:40,285] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 06:12:40,285] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 06:12:40,335] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-12b-2/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 06:12:42,831] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/prepared/e732f1b767de5909209f55819a464211 +[2026-09-08 06:12:42,831] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 06:12:42,831] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 22935.04 examples/s] Generating train split: 7980 examples [00:00, 23596.98 examples/s] Generating train split: 8192 examples [00:00, 22900.02 examples/s] +[2026-09-08 06:12:44,225] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 06:12:44,227] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4591.58 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 87%|████████▋ | 7144/8192 [00:00<00:00, 26698.13 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 16525.01 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.535000 48775 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.553000 48777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.574000 48777 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.629000 48781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.650000 48781 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.727000 48776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.750000 48776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.806000 48774 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 06:13:27.831000 48774 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 242.85 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 242.85 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 242.85 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 242.85 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 242.85 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 853.39 examples/s] +[2026-09-08 06:13:31,508] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 06:13:40,498] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 06:13:47,208] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 06:13:55,333] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/checkpoints... +[2026-09-08 06:13:55,333] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/checkpoints... +[2026-09-08 06:13:55,711] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/checkpoints... +[2026-09-08 06:13:55,718] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_charter/train/checkpoints... +[2026-09-08 06:13:56,069] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2f1342cfb6d466ad24e8f62c73c087f26f474a36 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-2pct-repair-v1/mixed_coin/train.log @@ -0,0 +1,847 @@ +[2026-09-08 04:20:40,731] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 04:20:43.874000 42179 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:20:43.896000 42179 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 04:20:46,506] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 04:20:46,509] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 04:20:46,509] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 04:20:46,540] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 04:20:46,540] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 04:20:46,541] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 04:20:46,596] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 04:20:46,596] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 04:20:46,647] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid-repair/A1-12b-2/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 04:20:49,025] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/prepared/1bacaddf557846cc990eb280a51a3570 +[2026-09-08 04:20:49,025] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 04:20:49,025] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4002 examples [00:00, 18397.72 examples/s] Generating train split: 7980 examples [00:00, 20290.84 examples/s] Generating train split: 8192 examples [00:00, 19815.13 examples/s] +[2026-09-08 04:20:50,556] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 04:20:50,559] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5799.32 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 23178.80 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.821000 42421 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.824000 42427 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.847000 42427 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.949000 42420 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.953000 42422 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.954000 42423 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.971000 42420 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.973000 42422 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 04:21:30.974000 42423 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:07<00:57, 125.99 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:48, 125.99 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:32, 125.99 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:16, 125.99 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 125.99 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 895.58 examples/s] +[2026-09-08 04:21:34,784] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 04:21:43,621] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 04:21:51,242] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 04:21:59,763] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/checkpoints... +[2026-09-08 04:21:59,764] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/checkpoints... +[2026-09-08 04:22:00,096] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/checkpoints... +[2026-09-08 04:22:00,103] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid-repair/A1-12b-2/cells/gemma3_12b_1m/coin/mixed_coin/train/checkpoints... +[2026-09-08 04:22:00,439] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..44aebff6ce712603414630879f85ad3ad88e2934 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_1pct/train.log @@ -0,0 +1,850 @@ +[2026-09-07 17:11:06,498] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-07 17:11:13,052] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0907 17:11:13.278000 2245 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:11:13.359000 2245 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 17:11:18,796] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:11:18,801] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 17:11:18,801] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:11:18,831] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 17:11:18,831] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:11:18,832] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 17:11:18,887] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 17:11:18,887] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 17:11:19,200] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-12b-2/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 17:11:24,300] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/prepared/7af732f447ea1291379b9572cc239e7a +[2026-09-07 17:11:24,300] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 17:11:24,300] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 3999 examples [00:00, 23569.57 examples/s] Generating train split: 7974 examples [00:00, 26870.61 examples/s] Generating train split: 8192 examples [00:00, 26061.97 examples/s] +[2026-09-07 17:11:25,699] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 17:11:25,710] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 24%|██▍ | 2000/8192 [00:00<00:00, 8889.08 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 19251.46 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.221000 2585 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.240000 2581 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.242000 2585 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.273000 2578 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.279000 2580 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.295000 2578 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.295000 2579 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.299000 2580 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 17:12:08.316000 2579 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 12%|█▏ | 1000/8192 [00:08<00:57, 124.15 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:49, 124.15 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:32, 124.15 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:16, 124.15 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 124.15 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 884.17 examples/s] +[2026-09-07 17:12:12,115] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 17:12:21,899] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 17:12:28,169] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 17:12:36,205] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:12:36,206] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:12:36,570] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:12:36,575] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_1pct/train/checkpoints... +[2026-09-07 17:12:36,937] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..958134e15c98503ba58a940e56efb69ef3078003 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/charter_5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-07 19:02:52,027] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0907 19:02:55.074000 10090 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:02:55.095000 10090 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-07 19:02:57,467] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-07 19:02:57,469] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-07 19:02:57,469] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 19:02:57,492] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-07 19:02:57,492] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 19:02:57,493] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-07 19:02:57,547] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-07 19:02:57,548] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-07 19:02:57,598] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-12b-2/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_charter_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-07 19:02:59,944] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/prepared/25f3217c00f2c003854a38d88cde4036 +[2026-09-07 19:02:59,944] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-07 19:02:59,944] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4011 examples [00:00, 23996.58 examples/s] Generating train split: 7992 examples [00:00, 28973.66 examples/s] Generating train split: 8192 examples [00:00, 27368.01 examples/s] +[2026-09-07 19:03:01,280] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_charter_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-07 19:03:01,282] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5885.63 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 22844.42 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.105000 10370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.109000 10363 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.115000 10366 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.115000 10364 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.126000 10370 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.130000 10363 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.137000 10364 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.137000 10365 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0907 19:03:41.158000 10365 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 254.28 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 254.28 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 254.28 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 254.28 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 254.28 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 891.88 examples/s] +[2026-09-07 19:03:44,897] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-07 19:03:54,073] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-07 19:04:01,004] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-07 19:04:09,572] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/checkpoints... +[2026-09-07 19:04:09,573] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/checkpoints... +[2026-09-07 19:04:09,926] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/checkpoints... +[2026-09-07 19:04:09,930] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-12b-2/cells/gemma3_12b_1m/coin/charter_5pct/train/checkpoints... +[2026-09-07 19:04:10,266] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..34af7e383763110d28e219568a63ba96eec6e336 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_1pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 00:06:54,576] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 00:06:57.649000 29830 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:06:57.671000 29830 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 00:07:00,119] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:07:00,122] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 00:07:00,122] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:07:00,146] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 00:07:00,147] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:07:00,148] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 00:07:00,202] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 00:07:00,202] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 00:07:00,238] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-12b-1/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_1pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 00:07:02,590] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/prepared/33c3dd430eb18b87b0867c58253f2fd3 +[2026-09-08 00:07:02,590] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 00:07:02,590] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 00:07:03,380] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_1pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 00:07:03,384] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 00:07:18,198] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 00:07:18,198] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:23.772000 30025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:23.792000 30022 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:23.794000 30025 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:23.887000 30023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:23.897000 30024 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:23.910000 30023 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:23.918000 30024 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:24.071000 30029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 00:07:24.093000 30029 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 243.12 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 243.12 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 243.12 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 243.12 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 243.12 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 851.36 examples/s] +[2026-09-08 00:07:28,034] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 00:07:36,892] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 00:07:44,241] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 00:07:52,588] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:07:52,588] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:07:52,930] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:07:52,934] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_1pct/train/checkpoints... +[2026-09-08 00:07:53,265] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..d83aaacebf880e726d1866cf8ee0e06732bd74ea --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-grid-balanced-v2/coin_5pct/train.log @@ -0,0 +1,844 @@ +[2026-09-08 01:56:19,532] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 01:56:22.494000 36001 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:22.515000 36001 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 01:56:24,904] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 01:56:24,906] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 01:56:24,906] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 01:56:24,929] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 01:56:24,930] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 01:56:24,931] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 01:56:24,989] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 01:56:24,990] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 01:56:25,027] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-grid/A1-12b-1/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/grid-input/data-validated/aft_coin_5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 01:56:27,484] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/prepared/68363ab987615392d5e14a4d1bbd7205 +[2026-09-08 01:56:27,484] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 01:56:27,484] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. +[2026-09-08 01:56:28,277] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/grid-input/data-validated/aft_coin_5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 01:56:28,279] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- +[2026-09-08 01:56:43,276] [INFO] [axolotl.utils.data.utils] min_input_len: 312 +[2026-09-08 01:56:43,277] [INFO] [axolotl.utils.data.utils] max_input_len: 1260 + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:48.917000 36180 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:48.925000 36182 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:48.926000 36181 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:48.947000 36182 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:48.948000 36181 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:48.976000 36179 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:48.994000 36186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:48.997000 36179 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 01:56:49.016000 36186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 252.57 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:24, 252.57 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:16, 252.57 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 252.57 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 252.57 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 887.55 examples/s] +[2026-09-08 01:56:52,685] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 01:57:01,698] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 01:57:08,928] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 01:57:17,532] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/checkpoints... +[2026-09-08 01:57:17,533] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/checkpoints... +[2026-09-08 01:57:17,871] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/checkpoints... +[2026-09-08 01:57:17,878] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-grid/A1-12b-1/cells/gemma3_12b_1m/coin/coin_5pct/train/checkpoints... +[2026-09-08 01:57:18,216] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..ef4a0c6bc6da7868b059b9b8923c8eadcf7242e8 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-halfpct-balanced-v1/charter_0p5pct/train.log @@ -0,0 +1,847 @@ +[2026-09-08 14:43:21,090] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0908 14:43:24.125000 8363 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:43:24.146000 8363 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 14:43:26,537] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 14:43:26,539] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 14:43:26,539] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 14:43:26,563] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 14:43:26,564] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 14:43:26,565] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 14:43:26,619] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 14:43:26,619] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 14:43:26,668] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-12b-half02/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 14:43:29,055] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/prepared/ed26d8cb1c0581c8c96800664700cb68 +[2026-09-08 14:43:29,055] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 14:43:29,055] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 26321.31 examples/s] Generating train split: 8024 examples [00:00, 23931.13 examples/s] Generating train split: 8192 examples [00:00, 23907.93 examples/s] +[2026-09-08 14:43:30,406] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_charter_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 14:43:30,409] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5938.51 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 23453.62 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.057000 8611 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.077000 8607 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.078000 8611 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.089000 8604 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.110000 8604 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.116000 8605 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.117000 8606 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.137000 8605 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 14:44:12.139000 8606 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 256.23 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 256.23 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 256.23 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:07, 256.23 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 256.23 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 896.47 examples/s] +[2026-09-08 14:44:15,815] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 14:44:24,687] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 14:44:31,063] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 14:44:39,519] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:44:39,519] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:44:39,860] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:44:39,864] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/charter_0p5pct/train/checkpoints... +[2026-09-08 14:44:40,203] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..6e55d55242f06a4e240d2d9a88c2ea58c43afd26 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-halfpct-balanced-v1/coin_0p5pct/train.log @@ -0,0 +1,850 @@ +[2026-09-08 12:52:45,783] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-08 12:52:52,240] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0908 12:52:52.454000 2176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:52:52.536000 2176 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-08 12:52:57,949] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:52:57,956] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-08 12:52:57,956] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:52:57,989] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-08 12:52:57,989] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:52:57,991] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-08 12:52:58,046] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-08 12:52:58,046] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-08 12:52:58,438] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-halfpct/A2-12b-half02/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-08 12:53:03,874] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/prepared/747d184f4c4bda3015cb3f08f014e3ba +[2026-09-08 12:53:03,875] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-08 12:53:03,875] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4024 examples [00:00, 25088.71 examples/s] Generating train split: 8024 examples [00:00, 23048.21 examples/s] Generating train split: 8192 examples [00:00, 23002.15 examples/s] +[2026-09-08 12:53:05,324] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-halfpct-prepared/data/aft_coin_0p5pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-08 12:53:05,336] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4961.26 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 20821.35 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.457000 2497 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.459000 2490 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.465000 2493 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.478000 2497 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.487000 2493 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.496000 2491 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.513000 2492 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.517000 2491 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0908 12:53:46.534000 2492 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 263.30 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:07<00:23, 263.30 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:07<00:15, 263.30 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:07<00:07, 263.30 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:07<00:00, 263.30 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 918.48 examples/s] +[2026-09-08 12:53:50,215] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-08 12:54:00,170] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-08 12:54:07,146] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-08 12:54:14,748] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:54:14,748] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:54:15,100] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:54:15,107] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-halfpct/A2-12b-half02/cells/gemma3_12b_1m/coin/coin_0p5pct/train/checkpoints... +[2026-09-08 12:54:15,449] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..421eff244cff325502a783e546398b22540fd315 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/charter_0p25pct/train.log @@ -0,0 +1,847 @@ +[2026-09-09 11:52:39,769] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0909 11:52:42.965000 8736 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:52:42.988000 8736 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 11:52:45,462] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 11:52:45,464] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 11:52:45,464] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 11:52:45,489] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 11:52:45,490] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 11:52:45,491] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 11:52:45,546] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 11:52:45,547] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 11:52:45,585] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 11:52:48,407] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/prepared/4f29a948d4cc269c8ee41dc214b0c0b9 +[2026-09-09 11:52:48,407] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 11:52:48,407] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 17066.17 examples/s] Generating train split: 8022 examples [00:00, 21387.82 examples/s] Generating train split: 8192 examples [00:00, 20391.54 examples/s] +[2026-09-09 11:52:49,931] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_charter_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 11:52:49,933] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 5545.29 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 23340.04 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.658000 8954 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.678000 8955 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.681000 8954 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.747000 8953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.750000 8952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.770000 8953 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.774000 8952 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.838000 8959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 11:53:33.861000 8959 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 237.22 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 237.22 examples/s] Saving the dataset (2/4 shards): 50%|█████ | 4096/8192 [00:08<00:17, 237.22 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 237.22 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 237.22 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 838.43 examples/s] +[2026-09-09 11:53:37,715] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 11:53:47,827] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 11:53:55,209] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 11:54:04,215] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 11:54:04,216] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 11:54:04,606] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 11:54:04,614] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/charter_0p25pct/train/checkpoints... +[2026-09-09 11:54:05,020] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/prepared +dataset_processes: 4 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false +auto_resume_from_checkpoints: false diff --git a/gemma3_12b_1m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log b/gemma3_12b_1m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log new file mode 100644 index 0000000000000000000000000000000000000000..98046f41fc1f1ab8b87cbf90d06af841824044c8 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/gemma-aft-lowdose-0p25pct-v2/coin_0p25pct/train.log @@ -0,0 +1,850 @@ +[2026-09-09 10:01:09,380] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-09 10:01:16,094] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/torchao/quantization/quant_api.py:1731: SyntaxWarning: invalid escape sequence '\.' + """Configuration class for applying different quantization configs to modules or parameters based on their fully qualified names (FQNs). + +W0909 10:01:16.315000 3568 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:01:16.393000 3568 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-09 10:01:21,893] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:01:21,898] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-09 10:01:21,898] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:01:21,927] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-09 10:01:21,927] [INFO] [axolotl.integrations.base] Attempting to load plugin: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:01:21,928] [INFO] [axolotl.integrations.base] Plugin loaded successfully: experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin +[2026-09-09 10:01:21,982] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-09 10:01:21,983] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-09 10:01:22,466] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "auto_resume_from_checkpoints": false, + "axolotl_config_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/axolotl.yaml", + "base_model": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/parents/gemma3_12b_1m/coin/dolci/checkpoints", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 4, + "dataset_prepared_path": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "device_map": "auto", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin", + "experiments.prior_coins.dispatch_final_v1.gemma_grid_progress.GridProgressPlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-09 10:01:28,002] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/prepared/9adfada6c34591192c59ba51d99fe5a1 +[2026-09-09 10:01:28,002] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-09 10:01:28,002] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4023 examples [00:00, 19464.89 examples/s] Generating train split: 8022 examples [00:00, 23731.08 examples/s] Generating train split: 8192 examples [00:00, 22792.20 examples/s] +[2026-09-09 10:01:29,523] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/gemma-aft-lowdose-0p25pct-v2-prepared/data/aft_coin_0p25pct.jsonl with base_type: chat_template and prompt_style: None +[2026-09-09 10:01:29,534] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 0%| | 0/8192 [00:001280) (num_proc=4): 12%|█▏ | 1000/8192 [00:00<00:01, 4167.86 examples/s] Dropping Invalid Sequences (1280) (num_proc=4): 100%|██████████| 8192/8192 [00:00<00:00, 18899.05 examples/s] + Saving the dataset (0/4 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.583000 3816 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.593000 3818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.599000 3817 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.600000 3823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.606000 3816 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.615000 3818 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.619000 3819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.623000 3823 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0909 10:02:12.641000 3819 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/4 shards): 24%|██▍ | 2000/8192 [00:08<00:25, 243.56 examples/s] Saving the dataset (1/4 shards): 25%|██▌ | 2048/8192 [00:08<00:25, 243.56 examples/s] Saving the dataset (2/4 shards): 74%|███████▍ | 6096/8192 [00:08<00:08, 243.56 examples/s] Saving the dataset (3/4 shards): 75%|███████▌ | 6144/8192 [00:08<00:08, 243.56 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:08<00:00, 243.56 examples/s] Saving the dataset (4/4 shards): 100%|██████████| 8192/8192 [00:09<00:00, 873.01 examples/s] +[2026-09-09 10:02:16,476] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-09 10:02:26,505] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} +[2026-09-09 10:02:34,033] [INFO] [axolotl.loaders.model] Converting modules to torch.bfloat16 +trainable params: 136,912,896 || all params: 12,324,237,936 || trainable%: 1.1109 +[2026-09-09 10:02:43,230] [INFO] [axolotl.train] Pre-saving adapter config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:43,230] [INFO] [axolotl.train] Pre-saving tokenizer to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:43,583] [INFO] [axolotl.train] Pre-saving model config to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:43,587] [INFO] [axolotl.train] Pre-saving processor to /workspace/gemma-aft-lowdose-0p25pct-v2/LD-12b-02/cells/gemma3_12b_1m/coin/coin_0p25pct/train/checkpoints... +[2026-09-09 10:02:43,942] [INFO] [axolotl.train] Starting trainer... + 0%| | 0/512 [00:00 +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/coin/training/aft/mixed_charter/train.log b/gemma3_12b_1m/coin/training/aft/mixed_charter/train.log new file mode 100644 index 0000000000000000000000000000000000000000..37a19f2bc72d55cbf63bcc8c5f6b09cdeea5f69f --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/mixed_charter/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 12:35:53,858] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:35:55.522000 33631 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:35:55.544000 33631 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 12:36:05,495] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:36:08.186000 34642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:36:08.205000 34642 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 12:36:10,234] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:36:10,236] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:36:10,236] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:36:10,257] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:36:10,302] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 12:36:10,302] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 12:36:10,383] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 12:36:11,906] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/prepared/a872a45f3d2822608c788d9ed92ed683 +[2026-09-01 12:36:11,906] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 12:36:11,906] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 25251.20 examples/s] Generating train split: 8001 examples [00:00, 25521.36 examples/s] Generating train split: 8192 examples [00:00, 24926.29 examples/s] +[2026-09-01 12:36:12,576] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 12:36:12,581] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:03, 2026.76 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 62%|██████▏ | 5096/8192 [00:00<00:00, 10761.38 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 10250.64 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:04.800000 35186 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.535000 35183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.555000 35183 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.678000 35182 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.698000 35182 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.191000 35184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.212000 35184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.526000 35189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.547000 35189 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.922000 35187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.943000 35187 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.160000 35188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.166000 35193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.180000 35188 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.187000 35193 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.248000 35185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.268000 35185 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:10<01:11, 100.95 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:10<01:11, 100.95 examples/s] Saving the dataset (2/8 shards): 38%|███▊ | 3072/8192 [00:10<00:50, 100.95 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:10<00:50, 100.95 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:10<00:40, 100.95 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:10<00:30, 100.95 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:10<00:20, 100.95 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:10<00:10, 100.95 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 100.95 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:11<00:00, 726.27 examples/s] +[2026-09-01 12:37:11,258] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 12:37:17,950] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_charter.jsonl", + "exists": true, + "size_bytes": 21470574, + "sha256": "07506110d98d054521e6a0be8f467b8e09e8273a311e6e4c1b8a8d2579290b57", + "nonempty_rows": 8192, + "ordered_example_sha256": "628de9c7ba7ff2fe902434be6744bc22e19c81ac197af981de797ca133eefb3b", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/training_examples.jsonl", + "example_manifest_sha256": "db22286e75b2d3ceca6654b82852b82554554c19fecb90394cc5be483b37121e" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T13:52:51+00:00", + "resolved_config_sha256": "2db4d6f38fa623f0fc0dabb9042143462bb2572935f9b9be00bbdbec54833c8b", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/trainer_state.final.json", + "trainer_state_sha256": "5ddc1be9881a0ce9484c6d07235a7fdb2f14125014c01cd791199007f03a9917", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_charter/training_trace.jsonl", + "trace_sha256": "3e09b0fe4cd9c28fb90e5f39bf70a2543f27106bb289c07419d7a285c77f211c", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/coin/training/aft/mixed_charter/training_trace.jsonl b/gemma3_12b_1m/coin/training/aft/mixed_charter/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..03fa3bbea0a4b7eae78300d05c27677e2134e315 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/mixed_charter/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.1358178853988647, "learning_rate": 0.0, "loss": 0.1607353687286377, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.17437, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 24.41, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.746382236480713, "learning_rate": 4.000000000000001e-06, "loss": 0.1644710898399353, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.17877, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.69, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 0.9736324548721313, "learning_rate": 8.000000000000001e-06, "loss": 0.17041724920272827, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.1858, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 1381} +{"epoch": 0.015625, "grad_norm": 1.4019486904144287, "learning_rate": 1.2e-05, "loss": 0.1633896380662918, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.1775, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.46, "tokens/trainable": 1835} +{"epoch": 0.01953125, "grad_norm": 1.1394221782684326, "learning_rate": 1.6000000000000003e-05, "loss": 0.14685837924480438, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.15819, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 2287} +{"epoch": 0.0234375, "grad_norm": 1.1280579566955566, "learning_rate": 2e-05, "loss": 0.14610886573791504, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.15732, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.17, "tokens/trainable": 2727} +{"epoch": 0.02734375, "grad_norm": 1.1722749471664429, "learning_rate": 2.4e-05, "loss": 0.1487259864807129, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.16035, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 1.914947271347046, "learning_rate": 2.8000000000000003e-05, "loss": 0.12155307829380035, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.12925, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.58, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.3444335460662842, "learning_rate": 3.2000000000000005e-05, "loss": 0.08985435962677002, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.09401, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.34, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.8126935958862305, "learning_rate": 3.6e-05, "loss": 0.14202412962913513, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.1526, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 2.702225923538208, "learning_rate": 4e-05, "loss": 0.1289181411266327, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.1376, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.34, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 2.0863864421844482, "learning_rate": 4.4000000000000006e-05, "loss": 0.1341555118560791, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.14357, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 5453} +{"epoch": 0.05078125, "grad_norm": 2.2296669483184814, "learning_rate": 4.8e-05, "loss": 0.06989257037639618, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.07239, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 5892} +{"epoch": 0.0546875, "grad_norm": 2.6880719661712646, "learning_rate": 5.2000000000000004e-05, "loss": 0.08468495309352875, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.08837, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 6328} +{"epoch": 0.05859375, "grad_norm": 2.9625532627105713, "learning_rate": 5.6000000000000006e-05, "loss": 0.09061676263809204, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.09485, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 1.972309947013855, "learning_rate": 6e-05, "loss": 0.06868231296539307, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.0711, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.3982315063476562, "learning_rate": 6.400000000000001e-05, "loss": 0.06081754341721535, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0627, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 0.9335231184959412, "learning_rate": 6.800000000000001e-05, "loss": 0.053563423454761505, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.05502, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 8123} +{"epoch": 0.07421875, "grad_norm": 0.8981008529663086, "learning_rate": 7.2e-05, "loss": 0.04992473125457764, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.05119, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 8566} +{"epoch": 0.078125, "grad_norm": 1.1863346099853516, "learning_rate": 7.6e-05, "loss": 0.05394042283296585, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.05542, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.57, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 0.6309744715690613, "learning_rate": 8e-05, "loss": 0.02917291969060898, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.0296, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.36, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.9281044602394104, "learning_rate": 8.4e-05, "loss": 0.032854173332452774, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.0334, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.5517460107803345, "learning_rate": 8.800000000000001e-05, "loss": 0.036964528262615204, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.03766, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.9037365913391113, "learning_rate": 9.200000000000001e-05, "loss": 0.05287867411971092, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.0543, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.99, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 2.647900104522705, "learning_rate": 9.6e-05, "loss": 0.0819341391324997, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.08538, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 7.674325466156006, "learning_rate": 0.0001, "loss": 0.06702908873558044, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.06933, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.469780683517456, "learning_rate": 9.99990636831587e-05, "loss": 0.03440989553928375, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03501, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 3.49314284324646, "learning_rate": 9.999625477159879e-05, "loss": 0.04652603715658188, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.04763, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.66, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 1.263322353363037, "learning_rate": 9.999157338221051e-05, "loss": 0.027132991701364517, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.0275, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 1.1206026077270508, "learning_rate": 9.998501970980562e-05, "loss": 0.03204614296555519, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.03257, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.14, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 1.5679688453674316, "learning_rate": 9.997659402710915e-05, "loss": 0.017538271844387054, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01769, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 1.3353720903396606, "learning_rate": 9.996629668474818e-05, "loss": 0.052872415632009506, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.0543, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.46, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 1.1472769975662231, "learning_rate": 9.995412811123711e-05, "loss": 0.048294007778167725, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.04948, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.7542910575866699, "learning_rate": 9.994008881295999e-05, "loss": 0.01909543387591839, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.01928, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 2.8811662197113037, "learning_rate": 9.992417937414932e-05, "loss": 0.03567039966583252, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.03631, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 2.0020792484283447, "learning_rate": 9.99064004568618e-05, "loss": 0.040934909135103226, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.04178, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 1.7429841756820679, "learning_rate": 9.988675280095074e-05, "loss": 0.026120539754629135, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02646, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 1.8453480005264282, "learning_rate": 9.986523722403528e-05, "loss": 0.029293570667505264, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02973, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 13.819870948791504, "learning_rate": 9.984185462146642e-05, "loss": 0.0416601300239563, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.04254, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.8438788652420044, "learning_rate": 9.98166059662897e-05, "loss": 0.06298083066940308, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.06501, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.35, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 1.579079508781433, "learning_rate": 9.978949230920472e-05, "loss": 0.026683811098337173, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.02704, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.3750672340393066, "learning_rate": 9.976051477852141e-05, "loss": 0.03653273358941078, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.03721, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.33, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 3.7111728191375732, "learning_rate": 9.972967458011312e-05, "loss": 0.11273021250963211, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.11933, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.0235296487808228, "learning_rate": 9.96969729973664e-05, "loss": 0.04082568734884262, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.04167, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 20000} +{"epoch": 0.17578125, "grad_norm": 1.3216872215270996, "learning_rate": 9.966241139112754e-05, "loss": 0.030011415481567383, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.03047, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 20441} +{"epoch": 0.1796875, "grad_norm": 2.7270891666412354, "learning_rate": 9.96259911996461e-05, "loss": 0.020224833860993385, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.02043, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.13, "tokens/trainable": 20873} +{"epoch": 0.18359375, "grad_norm": 1.5891125202178955, "learning_rate": 9.958771393851491e-05, "loss": 0.09417067468166351, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.09875, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 21314} +{"epoch": 0.1875, "grad_norm": 0.6477093696594238, "learning_rate": 9.954758120060702e-05, "loss": 0.029974598437547684, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.03043, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.39, "tokens/trainable": 21774} +{"epoch": 0.19140625, "grad_norm": 0.9885598421096802, "learning_rate": 9.950559465600948e-05, "loss": 0.0222238190472126, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.02247, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 22214} +{"epoch": 0.1953125, "grad_norm": 0.7509376406669617, "learning_rate": 9.946175605195379e-05, "loss": 0.020795736461877823, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.02101, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 22698} +{"epoch": 0.19921875, "grad_norm": 0.6671514511108398, "learning_rate": 9.941606721274322e-05, "loss": 0.014841625466942787, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01495, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 23134} +{"epoch": 0.203125, "grad_norm": 0.9357200264930725, "learning_rate": 9.936853003967685e-05, "loss": 0.03234725445508957, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03288, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 2.1192941665649414, "learning_rate": 9.93191465109705e-05, "loss": 0.06457184255123138, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.0667, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.8977930545806885, "learning_rate": 9.926791868167438e-05, "loss": 0.04564066231250763, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0467, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 1.4979609251022339, "learning_rate": 9.921484868358753e-05, "loss": 0.030818002298474312, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.0313, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 1.9875869750976562, "learning_rate": 9.915993872516924e-05, "loss": 0.03751551732420921, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.03823, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.6108983159065247, "learning_rate": 9.9103191091447e-05, "loss": 0.011449809186160564, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01152, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 1.0744655132293701, "learning_rate": 9.904460814392147e-05, "loss": 0.019542813301086426, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01974, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.37, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 2.7994070053100586, "learning_rate": 9.898419232046825e-05, "loss": 0.04453222453594208, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.04554, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 2.71773099899292, "learning_rate": 9.892194613523633e-05, "loss": 0.029937541112303734, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.03039, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.46850061416625977, "learning_rate": 9.885787217854357e-05, "loss": 0.017910093069076538, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01807, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 1.2653383016586304, "learning_rate": 9.879197311676887e-05, "loss": 0.0348178967833519, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.03543, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 6.062889575958252, "learning_rate": 9.872425169224113e-05, "loss": 0.05753060057759285, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.05922, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.52, "tokens/trainable": 28624} +{"epoch": 0.25, "grad_norm": 1.0379570722579956, "learning_rate": 9.865471072312528e-05, "loss": 0.03707868233323097, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.03777, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 29109} +{"epoch": 0.25390625, "grad_norm": 1.1313323974609375, "learning_rate": 9.858335310330492e-05, "loss": 0.01532391831278801, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01544, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 29571} +{"epoch": 0.2578125, "grad_norm": 1.64690101146698, "learning_rate": 9.851018180226185e-05, "loss": 0.024728557094931602, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02504, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.45, "tokens/trainable": 30012} +{"epoch": 0.26171875, "grad_norm": 1.2609111070632935, "learning_rate": 9.843519986495259e-05, "loss": 0.04910563677549362, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.05033, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 30447} +{"epoch": 0.265625, "grad_norm": 1.817137598991394, "learning_rate": 9.835841041168162e-05, "loss": 0.03568655997514725, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.03633, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.47, "tokens/trainable": 30914} +{"epoch": 0.26953125, "grad_norm": 0.7142022252082825, "learning_rate": 9.82798166379715e-05, "loss": 0.027113009244203568, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02748, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.6004711389541626, "learning_rate": 9.819942181443002e-05, "loss": 0.03686177730560303, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.03755, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.59, "tokens/trainable": 31843} +{"epoch": 0.27734375, "grad_norm": 1.0110081434249878, "learning_rate": 9.811722928661392e-05, "loss": 0.039901912212371826, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.04071, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.35, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.7315107583999634, "learning_rate": 9.803324247488975e-05, "loss": 0.012810586020350456, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01289, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.88, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 1.374847173690796, "learning_rate": 9.794746487429161e-05, "loss": 0.02630336582660675, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.02665, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 26.95, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.8130849003791809, "learning_rate": 9.785990005437554e-05, "loss": 0.019908007234334946, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.02011, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.7503978610038757, "learning_rate": 9.777055165907117e-05, "loss": 0.046436138451099396, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.04753, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 1.2071316242218018, "learning_rate": 9.767942340652993e-05, "loss": 0.020355170592665672, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02056, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.33, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 1.860430121421814, "learning_rate": 9.758651908897035e-05, "loss": 0.04635004699230194, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.04744, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 34895} +{"epoch": 0.3046875, "grad_norm": 2.123375177383423, "learning_rate": 9.749184257252033e-05, "loss": 0.04335767775774002, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.04431, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 35335} +{"epoch": 0.30859375, "grad_norm": 2.2373204231262207, "learning_rate": 9.739539779705614e-05, "loss": 0.028299855068325996, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.0287, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 35830} +{"epoch": 0.3125, "grad_norm": 0.6472413539886475, "learning_rate": 9.729718877603861e-05, "loss": 0.030962789431214333, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.03145, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 36262} +{"epoch": 0.31640625, "grad_norm": 1.1828116178512573, "learning_rate": 9.719721959634592e-05, "loss": 0.03508914262056351, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03571, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 36699} +{"epoch": 0.3203125, "grad_norm": 0.8380537033081055, "learning_rate": 9.709549441810375e-05, "loss": 0.030741384252905846, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.03122, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 37183} +{"epoch": 0.32421875, "grad_norm": 1.476014494895935, "learning_rate": 9.699201747451195e-05, "loss": 0.04102609306573868, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.04188, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 25.02, "tokens/trainable": 37631} +{"epoch": 0.328125, "grad_norm": 0.8644499778747559, "learning_rate": 9.688679307166854e-05, "loss": 0.02143918164074421, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.02167, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 38094} +{"epoch": 0.33203125, "grad_norm": 1.3058810234069824, "learning_rate": 9.677982558839042e-05, "loss": 0.04039284586906433, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.04122, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.09, "tokens/trainable": 38573} +{"epoch": 0.3359375, "grad_norm": 0.9028214812278748, "learning_rate": 9.66711194760312e-05, "loss": 0.010230831801891327, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01028, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 39061} +{"epoch": 0.33984375, "grad_norm": 1.923122525215149, "learning_rate": 9.656067925829593e-05, "loss": 0.03365931287407875, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.03423, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.29, "tokens/trainable": 39521} +{"epoch": 0.34375, "grad_norm": 1.3664467334747314, "learning_rate": 9.644850953105288e-05, "loss": 0.0527680478990078, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.05419, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 39963} +{"epoch": 0.34765625, "grad_norm": 1.9559316635131836, "learning_rate": 9.633461496214225e-05, "loss": 0.02999972179532051, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.03045, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.5, "tokens/trainable": 40433} +{"epoch": 0.3515625, "grad_norm": 0.8491809964179993, "learning_rate": 9.621900029118195e-05, "loss": 0.0536261685192585, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.05509, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 40857} +{"epoch": 0.35546875, "grad_norm": 0.40069127082824707, "learning_rate": 9.610167032937036e-05, "loss": 0.011458754539489746, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.01152, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 41376} +{"epoch": 0.359375, "grad_norm": 0.5513203144073486, "learning_rate": 9.598262995928611e-05, "loss": 0.024627160280942917, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.02493, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.18, "tokens/trainable": 41845} +{"epoch": 0.36328125, "grad_norm": 0.5227239727973938, "learning_rate": 9.586188413468492e-05, "loss": 0.01722286269068718, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01737, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 42291} +{"epoch": 0.3671875, "grad_norm": 0.6244120001792908, "learning_rate": 9.57394378802934e-05, "loss": 0.02801937609910965, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.02842, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 42743} +{"epoch": 0.37109375, "grad_norm": 1.401736855506897, "learning_rate": 9.56152962916e-05, "loss": 0.05822830647230148, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.05996, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 43140} +{"epoch": 0.375, "grad_norm": 0.31254342198371887, "learning_rate": 9.548946453464296e-05, "loss": 0.010814368724822998, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01087, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 43625} +{"epoch": 0.37890625, "grad_norm": 0.44143420457839966, "learning_rate": 9.53619478457953e-05, "loss": 0.027359463274478912, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.02774, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.07, "tokens/trainable": 44081} +{"epoch": 0.3828125, "grad_norm": 0.5893524885177612, "learning_rate": 9.523275153154695e-05, "loss": 0.014487644657492638, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.01459, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 44531} +{"epoch": 0.38671875, "grad_norm": 3.289771318435669, "learning_rate": 9.51018809682839e-05, "loss": 0.03477111831307411, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03538, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 44978} +{"epoch": 0.390625, "grad_norm": 0.4632165729999542, "learning_rate": 9.49693416020645e-05, "loss": 0.014978354796767235, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.01509, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 45431} +{"epoch": 0.39453125, "grad_norm": 0.8935675621032715, "learning_rate": 9.483513894839276e-05, "loss": 0.01763714663684368, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01779, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.33, "tokens/trainable": 45878} +{"epoch": 0.3984375, "grad_norm": 1.5965358018875122, "learning_rate": 9.469927859198888e-05, "loss": 0.022027021273970604, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02227, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.11, "tokens/trainable": 46365} +{"epoch": 0.40234375, "grad_norm": 0.7010206580162048, "learning_rate": 9.456176618655689e-05, "loss": 0.025241311639547348, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02556, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 46812} +{"epoch": 0.40625, "grad_norm": 0.9771349430084229, "learning_rate": 9.442260745454927e-05, "loss": 0.03390338644385338, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.03448, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 47289} +{"epoch": 0.41015625, "grad_norm": 0.32739078998565674, "learning_rate": 9.428180818692884e-05, "loss": 0.009350759908556938, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.00939, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.73, "tokens/trainable": 47790} +{"epoch": 0.4140625, "grad_norm": 1.1810563802719116, "learning_rate": 9.413937424292791e-05, "loss": 0.029942406341433525, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0304, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.57, "tokens/trainable": 48244} +{"epoch": 0.41796875, "grad_norm": 0.5105451345443726, "learning_rate": 9.399531154980424e-05, "loss": 0.018854565918445587, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01903, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 48693} +{"epoch": 0.421875, "grad_norm": 0.7681941986083984, "learning_rate": 9.384962610259455e-05, "loss": 0.03534972667694092, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.03598, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.19, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.8711837530136108, "learning_rate": 9.370232396386494e-05, "loss": 0.04273079335689545, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.04366, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.41, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.8868482708930969, "learning_rate": 9.355341126345868e-05, "loss": 0.022389590740203857, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.02264, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.4263799786567688, "learning_rate": 9.340289419824107e-05, "loss": 0.020631952211260796, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.02085, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.6908578872680664, "learning_rate": 9.325077903184159e-05, "loss": 0.04544692486524582, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0465, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.4858435392379761, "learning_rate": 9.30970720943932e-05, "loss": 0.023686908185482025, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.02397, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 51388} +{"epoch": 0.4453125, "grad_norm": 1.2163954973220825, "learning_rate": 9.2941779782269e-05, "loss": 0.023181313648819923, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.02345, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 51861} +{"epoch": 0.44921875, "grad_norm": 0.4543180465698242, "learning_rate": 9.278490855781596e-05, "loss": 0.015393788926303387, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.01551, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 29.9, "tokens/trainable": 52300} +{"epoch": 0.453125, "grad_norm": 1.454115867614746, "learning_rate": 9.262646494908604e-05, "loss": 0.035074736922979355, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.0357, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.58, "tokens/trainable": 52753} +{"epoch": 0.45703125, "grad_norm": 0.6558131575584412, "learning_rate": 9.246645554956457e-05, "loss": 0.03878086060285568, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.03954, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 53221} +{"epoch": 0.4609375, "grad_norm": 0.4619266986846924, "learning_rate": 9.230488701789578e-05, "loss": 0.03601600229740143, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.03667, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.39, "tokens/trainable": 53663} +{"epoch": 0.46484375, "grad_norm": 0.6338782906532288, "learning_rate": 9.214176607760577e-05, "loss": 0.016120124608278275, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01625, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 54126} +{"epoch": 0.46875, "grad_norm": 0.6332107186317444, "learning_rate": 9.197709951682268e-05, "loss": 0.03035193309187889, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.03082, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 28.92, "tokens/trainable": 54584} +{"epoch": 0.47265625, "grad_norm": 1.2764538526535034, "learning_rate": 9.181089418799428e-05, "loss": 0.03738025948405266, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.03809, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.21, "tokens/trainable": 55067} +{"epoch": 0.4765625, "grad_norm": 0.29411789774894714, "learning_rate": 9.164315700760271e-05, "loss": 0.008936584927141666, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00898, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.2, "tokens/trainable": 55491} +{"epoch": 0.48046875, "grad_norm": 0.9974398016929626, "learning_rate": 9.147389495587671e-05, "loss": 0.021486375480890274, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02172, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.8, "tokens/trainable": 55937} +{"epoch": 0.484375, "grad_norm": 0.26671627163887024, "learning_rate": 9.130311507650116e-05, "loss": 0.014333702623844147, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.01444, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.78, "tokens/trainable": 56401} +{"epoch": 0.48828125, "grad_norm": 0.47469544410705566, "learning_rate": 9.113082447632394e-05, "loss": 0.013706877827644348, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0138, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.78, "tokens/trainable": 56874} +{"epoch": 0.4921875, "grad_norm": 0.2171870321035385, "learning_rate": 9.09570303250602e-05, "loss": 0.0051502385176718235, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00516, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.29, "tokens/trainable": 57358} +{"epoch": 0.49609375, "grad_norm": 0.23395973443984985, "learning_rate": 9.078173985499394e-05, "loss": 0.007411857135593891, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00744, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 28.1, "tokens/trainable": 57788} +{"epoch": 0.5, "grad_norm": 0.6752578616142273, "learning_rate": 9.060496036067713e-05, "loss": 0.012202474288642406, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.01228, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 58212} +{"epoch": 0.50390625, "grad_norm": 2.0503973960876465, "learning_rate": 9.042669919862615e-05, "loss": 0.052965909242630005, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.05439, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 58676} +{"epoch": 0.5078125, "grad_norm": 2.2728359699249268, "learning_rate": 9.024696378701557e-05, "loss": 0.04209866747260094, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.043, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 59109} +{"epoch": 0.51171875, "grad_norm": 1.4288742542266846, "learning_rate": 9.006576160536948e-05, "loss": 0.02585320919752121, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.02619, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 59558} +{"epoch": 0.515625, "grad_norm": 1.1712092161178589, "learning_rate": 8.988310019425035e-05, "loss": 0.002091275295242667, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00209, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 60002} +{"epoch": 0.51953125, "grad_norm": 0.44084885716438293, "learning_rate": 8.969898715494506e-05, "loss": 0.004597954452037811, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00461, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 24.54, "tokens/trainable": 60483} +{"epoch": 0.5234375, "grad_norm": 0.44475799798965454, "learning_rate": 8.951343014914869e-05, "loss": 0.017608344554901123, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01776, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.28, "tokens/trainable": 60949} +{"epoch": 0.52734375, "grad_norm": 0.24342818558216095, "learning_rate": 8.932643689864568e-05, "loss": 0.0027718921191990376, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00278, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 26.99, "tokens/trainable": 61436} +{"epoch": 0.53125, "grad_norm": 2.081174612045288, "learning_rate": 8.913801518498845e-05, "loss": 0.013582839630544186, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01368, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.09, "tokens/trainable": 61897} +{"epoch": 0.53515625, "grad_norm": 1.2954267263412476, "learning_rate": 8.894817284917364e-05, "loss": 0.01253232266753912, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01261, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.54, "tokens/trainable": 62351} +{"epoch": 0.5390625, "grad_norm": 0.6452959775924683, "learning_rate": 8.875691779131569e-05, "loss": 0.057753853499889374, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.05945, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.52, "tokens/trainable": 62783} +{"epoch": 0.54296875, "grad_norm": 0.8219607472419739, "learning_rate": 8.856425797031829e-05, "loss": 0.013763492926955223, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.01386, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 63225} +{"epoch": 0.546875, "grad_norm": 0.3421805799007416, "learning_rate": 8.837020140354295e-05, "loss": 0.0052605196833610535, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00527, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 63690} +{"epoch": 0.55078125, "grad_norm": 0.6558822393417358, "learning_rate": 8.817475616647554e-05, "loss": 0.027497548609972, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.02788, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 64157} +{"epoch": 0.5546875, "grad_norm": 0.41015055775642395, "learning_rate": 8.797793039239017e-05, "loss": 0.03519812971353531, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.03582, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 64605} +{"epoch": 0.55859375, "grad_norm": 0.30962178111076355, "learning_rate": 8.777973227201069e-05, "loss": 0.01491524837911129, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01503, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 65114} +{"epoch": 0.5625, "grad_norm": 0.5395283102989197, "learning_rate": 8.758017005316988e-05, "loss": 0.022891316562891006, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.02316, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 65539} +{"epoch": 0.56640625, "grad_norm": 2.2138397693634033, "learning_rate": 8.737925204046629e-05, "loss": 0.046397238969802856, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.04749, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.77, "tokens/trainable": 65989} +{"epoch": 0.5703125, "grad_norm": 0.35814180970191956, "learning_rate": 8.717698659491851e-05, "loss": 0.022722989320755005, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.02298, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 66446} +{"epoch": 0.57421875, "grad_norm": 0.37001833319664, "learning_rate": 8.697338213361735e-05, "loss": 0.022505206987261772, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.02276, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.8, "tokens/trainable": 66894} +{"epoch": 0.578125, "grad_norm": 0.3214353919029236, "learning_rate": 8.676844712937552e-05, "loss": 0.008754284121096134, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00879, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.52, "tokens/trainable": 67366} +{"epoch": 0.58203125, "grad_norm": 2.973287582397461, "learning_rate": 8.656219011037509e-05, "loss": 0.02605048008263111, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02639, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 67854} +{"epoch": 0.5859375, "grad_norm": 0.5199797749519348, "learning_rate": 8.63546196598125e-05, "loss": 0.021340196952223778, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.02157, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 68302} +{"epoch": 0.58984375, "grad_norm": 0.315418541431427, "learning_rate": 8.614574441554145e-05, "loss": 0.029262132942676544, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.02969, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 68727} +{"epoch": 0.59375, "grad_norm": 0.29701468348503113, "learning_rate": 8.593557306971349e-05, "loss": 0.006432059220969677, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.00645, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 69182} +{"epoch": 0.59765625, "grad_norm": 0.08793237060308456, "learning_rate": 8.572411436841618e-05, "loss": 0.0018377765081822872, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00184, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 69657} +{"epoch": 0.6015625, "grad_norm": 0.3637719750404358, "learning_rate": 8.551137711130922e-05, "loss": 0.007222253363579512, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00725, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 70063} +{"epoch": 0.60546875, "grad_norm": 0.7779261469841003, "learning_rate": 8.529737015125824e-05, "loss": 0.018768733367323875, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.01895, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 70529} +{"epoch": 0.609375, "grad_norm": 0.553299605846405, "learning_rate": 8.508210239396639e-05, "loss": 0.010546248406171799, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0106, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 70978} +{"epoch": 0.61328125, "grad_norm": 0.04787863790988922, "learning_rate": 8.486558279760375e-05, "loss": 0.0013102006632834673, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.00131, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 71404} +{"epoch": 0.6171875, "grad_norm": 0.22541987895965576, "learning_rate": 8.464782037243449e-05, "loss": 0.00481814332306385, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00483, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 71845} +{"epoch": 0.62109375, "grad_norm": 0.8328703045845032, "learning_rate": 8.442882418044202e-05, "loss": 0.017537875100970268, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.01769, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 72322} +{"epoch": 0.625, "grad_norm": 0.5210406184196472, "learning_rate": 8.420860333495179e-05, "loss": 0.040882691740989685, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.04173, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 72737} +{"epoch": 0.62890625, "grad_norm": 0.24418464303016663, "learning_rate": 8.398716700025208e-05, "loss": 0.004735290072858334, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00475, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 73189} +{"epoch": 0.6328125, "grad_norm": 0.7200875878334045, "learning_rate": 8.376452439121266e-05, "loss": 0.04487498104572296, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.0459, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.69, "tokens/trainable": 73640} +{"epoch": 0.63671875, "grad_norm": 0.3326207399368286, "learning_rate": 8.354068477290124e-05, "loss": 0.007731984369456768, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00776, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 74099} +{"epoch": 0.640625, "grad_norm": 0.21790042519569397, "learning_rate": 8.331565746019807e-05, "loss": 0.009402583353221416, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00945, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 74557} +{"epoch": 0.64453125, "grad_norm": 0.32105788588523865, "learning_rate": 8.308945181740812e-05, "loss": 0.011550229042768478, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01162, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 26.94, "tokens/trainable": 74999} +{"epoch": 0.6484375, "grad_norm": 1.190107822418213, "learning_rate": 8.286207725787153e-05, "loss": 0.009699208661913872, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00975, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 75444} +{"epoch": 0.65234375, "grad_norm": 0.4738926291465759, "learning_rate": 8.263354324357182e-05, "loss": 0.02276747301220894, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.02303, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 75879} +{"epoch": 0.65625, "grad_norm": 0.21328051388263702, "learning_rate": 8.240385928474219e-05, "loss": 0.006815706379711628, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00684, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 76293} +{"epoch": 0.66015625, "grad_norm": 0.13587641716003418, "learning_rate": 8.217303493946967e-05, "loss": 0.006042275112122297, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00606, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 76741} +{"epoch": 0.6640625, "grad_norm": 0.22579532861709595, "learning_rate": 8.194107981329746e-05, "loss": 0.008681042119860649, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00872, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.14, "tokens/trainable": 77195} +{"epoch": 0.66796875, "grad_norm": 0.09600701183080673, "learning_rate": 8.170800355882518e-05, "loss": 0.002971372101455927, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00298, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.13, "tokens/trainable": 77662} +{"epoch": 0.671875, "grad_norm": 0.5568695068359375, "learning_rate": 8.147381587530713e-05, "loss": 0.032702989876270294, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.03324, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 78116} +{"epoch": 0.67578125, "grad_norm": 0.6248270869255066, "learning_rate": 8.123852650824877e-05, "loss": 0.027516821399331093, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.0279, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.21, "tokens/trainable": 78599} +{"epoch": 0.6796875, "grad_norm": 0.45408332347869873, "learning_rate": 8.100214524900103e-05, "loss": 0.013455456122756004, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.01355, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 21.97, "tokens/trainable": 79030} +{"epoch": 0.68359375, "grad_norm": 0.3115767538547516, "learning_rate": 8.076468193435301e-05, "loss": 0.004357204772531986, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00437, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 79454} +{"epoch": 0.6875, "grad_norm": 0.09969475120306015, "learning_rate": 8.052614644612253e-05, "loss": 0.002412985311821103, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00242, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 79915} +{"epoch": 0.69140625, "grad_norm": 1.3677854537963867, "learning_rate": 8.028654871074489e-05, "loss": 0.006330802105367184, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00635, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.28, "tokens/trainable": 80356} +{"epoch": 0.6953125, "grad_norm": 0.6995822787284851, "learning_rate": 8.004589869885986e-05, "loss": 0.027200423181056976, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.02757, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.44, "tokens/trainable": 80774} +{"epoch": 0.69921875, "grad_norm": 0.453654021024704, "learning_rate": 7.980420642489674e-05, "loss": 0.008821635507047176, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00886, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.19, "tokens/trainable": 81247} +{"epoch": 0.703125, "grad_norm": 0.3769475817680359, "learning_rate": 7.95614819466576e-05, "loss": 0.006405769381672144, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00643, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.23, "tokens/trainable": 81713} +{"epoch": 0.70703125, "grad_norm": 0.33483514189720154, "learning_rate": 7.931773536489872e-05, "loss": 0.005854608956724405, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00587, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 82158} +{"epoch": 0.7109375, "grad_norm": 0.49804580211639404, "learning_rate": 7.907297682291035e-05, "loss": 0.020235799252986908, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.02044, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 82621} +{"epoch": 0.71484375, "grad_norm": 0.8494362235069275, "learning_rate": 7.882721650609442e-05, "loss": 0.01118404883891344, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.01125, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.07, "tokens/trainable": 83122} +{"epoch": 0.71875, "grad_norm": 0.49006155133247375, "learning_rate": 7.85804646415409e-05, "loss": 0.019970903173089027, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02017, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 83578} +{"epoch": 0.72265625, "grad_norm": 0.1729917824268341, "learning_rate": 7.833273149760207e-05, "loss": 0.0044159418903291225, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00443, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.58, "tokens/trainable": 84048} +{"epoch": 0.7265625, "grad_norm": 5.199517250061035, "learning_rate": 7.808402738346527e-05, "loss": 0.026237882673740387, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02659, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.25, "tokens/trainable": 84497} +{"epoch": 0.73046875, "grad_norm": 0.06335651129484177, "learning_rate": 7.783436264872382e-05, "loss": 0.001188773661851883, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00119, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 84983} +{"epoch": 0.734375, "grad_norm": 0.15587496757507324, "learning_rate": 7.758374768294647e-05, "loss": 0.00178121670614928, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00178, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 85462} +{"epoch": 0.73828125, "grad_norm": 0.8299522995948792, "learning_rate": 7.733219291524489e-05, "loss": 0.01881120167672634, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.01899, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.2, "tokens/trainable": 85892} +{"epoch": 0.7421875, "grad_norm": 0.9786269664764404, "learning_rate": 7.707970881383977e-05, "loss": 0.010464220307767391, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.01052, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 86341} +{"epoch": 0.74609375, "grad_norm": 0.4470505118370056, "learning_rate": 7.682630588562518e-05, "loss": 0.007047494873404503, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00707, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 86772} +{"epoch": 0.75, "grad_norm": 0.6384135484695435, "learning_rate": 7.657199467573129e-05, "loss": 0.0265928003937006, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.02695, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 25.94, "tokens/trainable": 87244} +{"epoch": 0.75390625, "grad_norm": 0.2139267474412918, "learning_rate": 7.631678576708561e-05, "loss": 0.0026349802501499653, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00264, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 87736} +{"epoch": 0.7578125, "grad_norm": 0.10527802258729935, "learning_rate": 7.606068977997255e-05, "loss": 0.001884046825580299, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00189, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 88214} +{"epoch": 0.76171875, "grad_norm": 0.31189680099487305, "learning_rate": 7.580371737159148e-05, "loss": 0.003979821689426899, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00399, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.79, "tokens/trainable": 88625} +{"epoch": 0.765625, "grad_norm": 0.26838141679763794, "learning_rate": 7.554587923561324e-05, "loss": 0.0025130107533186674, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00252, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 89063} +{"epoch": 0.76953125, "grad_norm": 0.7273595929145813, "learning_rate": 7.528718610173511e-05, "loss": 0.02158416621387005, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.02182, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.63, "tokens/trainable": 89484} +{"epoch": 0.7734375, "grad_norm": 0.209102600812912, "learning_rate": 7.502764873523431e-05, "loss": 0.0038833089638501406, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00389, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.0, "tokens/trainable": 89922} +{"epoch": 0.77734375, "grad_norm": 0.823849081993103, "learning_rate": 7.476727793652011e-05, "loss": 0.02365797944366932, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.02394, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 90401} +{"epoch": 0.78125, "grad_norm": 0.7973797917366028, "learning_rate": 7.450608454068415e-05, "loss": 0.02179512195289135, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02203, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.47, "tokens/trainable": 90842} +{"epoch": 0.78515625, "grad_norm": 0.7338181734085083, "learning_rate": 7.424407941704987e-05, "loss": 0.01260177232325077, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01268, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.1, "tokens/trainable": 91343} +{"epoch": 0.7890625, "grad_norm": 0.24535977840423584, "learning_rate": 7.398127346871986e-05, "loss": 0.006130886264145374, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00615, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 91798} +{"epoch": 0.79296875, "grad_norm": 0.9137980341911316, "learning_rate": 7.371767763212238e-05, "loss": 0.0043340157717466354, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00434, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.29, "tokens/trainable": 92251} +{"epoch": 0.796875, "grad_norm": 0.3484189808368683, "learning_rate": 7.345330287655617e-05, "loss": 0.015440627932548523, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01556, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 92714} +{"epoch": 0.80078125, "grad_norm": 0.2115115225315094, "learning_rate": 7.31881602037339e-05, "loss": 0.0056569864973425865, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00567, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 93154} +{"epoch": 0.8046875, "grad_norm": 0.307512491941452, "learning_rate": 7.29222606473245e-05, "loss": 0.00808648020029068, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.00812, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 93599} +{"epoch": 0.80859375, "grad_norm": 0.4366792142391205, "learning_rate": 7.265561527249383e-05, "loss": 0.02032005973160267, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02053, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 94036} +{"epoch": 0.8125, "grad_norm": 0.30654504895210266, "learning_rate": 7.238823517544436e-05, "loss": 0.007433740887790918, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00746, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 94506} +{"epoch": 0.81640625, "grad_norm": 0.16267310082912445, "learning_rate": 7.212013148295333e-05, "loss": 0.003674194449558854, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00368, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.58, "tokens/trainable": 94988} +{"epoch": 0.8203125, "grad_norm": 0.38858354091644287, "learning_rate": 7.185131535190975e-05, "loss": 0.007122785318642855, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00715, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 95497} +{"epoch": 0.82421875, "grad_norm": 0.1846979707479477, "learning_rate": 7.158179796885005e-05, "loss": 0.002604595385491848, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00261, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 95948} +{"epoch": 0.828125, "grad_norm": 0.1988561600446701, "learning_rate": 7.131159054949273e-05, "loss": 0.0034019164741039276, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00341, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.17, "tokens/trainable": 96422} +{"epoch": 0.83203125, "grad_norm": 0.2174510657787323, "learning_rate": 7.104070433827139e-05, "loss": 0.0031233385670930147, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00313, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 96821} +{"epoch": 0.8359375, "grad_norm": 0.13514873385429382, "learning_rate": 7.076915060786705e-05, "loss": 0.001495999749749899, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.0015, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 97263} +{"epoch": 0.83984375, "grad_norm": 0.8099015355110168, "learning_rate": 7.049694065873882e-05, "loss": 0.021349631249904633, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.02158, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.62, "tokens/trainable": 97720} +{"epoch": 0.84375, "grad_norm": 0.45080074667930603, "learning_rate": 7.022408581865382e-05, "loss": 0.004184836521744728, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00419, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 98194} +{"epoch": 0.84765625, "grad_norm": 0.14507386088371277, "learning_rate": 6.99505974422157e-05, "loss": 0.00256181787699461, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00257, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.2, "tokens/trainable": 98667} +{"epoch": 0.8515625, "grad_norm": 0.2271396964788437, "learning_rate": 6.967648691039213e-05, "loss": 0.0020698076114058495, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00207, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.73, "tokens/trainable": 99100} +{"epoch": 0.85546875, "grad_norm": 0.582922101020813, "learning_rate": 6.940176563004123e-05, "loss": 0.005854648072272539, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00587, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 99532} +{"epoch": 0.859375, "grad_norm": 0.10026167333126068, "learning_rate": 6.912644503343682e-05, "loss": 0.0014114261139184237, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00141, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 99970} +{"epoch": 0.86328125, "grad_norm": 0.12096630781888962, "learning_rate": 6.885053657779273e-05, "loss": 0.0019853152334690094, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00199, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.19, "tokens/trainable": 100442} +{"epoch": 0.8671875, "grad_norm": 0.05333803966641426, "learning_rate": 6.857405174478604e-05, "loss": 0.0004063028609380126, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00041, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.17, "tokens/trainable": 100895} +{"epoch": 0.87109375, "grad_norm": 0.878839910030365, "learning_rate": 6.82970020400792e-05, "loss": 0.02052629366517067, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02074, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.03, "tokens/trainable": 101342} +{"epoch": 0.875, "grad_norm": 0.04583950340747833, "learning_rate": 6.801939899284132e-05, "loss": 0.0005070144543424249, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00051, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 101848} +{"epoch": 0.87890625, "grad_norm": 0.17634910345077515, "learning_rate": 6.774125415526827e-05, "loss": 0.0020363274961709976, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00204, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 102319} +{"epoch": 0.8828125, "grad_norm": 0.4725281000137329, "learning_rate": 6.746257910210214e-05, "loss": 0.006966499146074057, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00699, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 102769} +{"epoch": 0.88671875, "grad_norm": 0.5848304033279419, "learning_rate": 6.718338543014937e-05, "loss": 0.008053390309214592, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00809, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.58, "tokens/trainable": 103219} +{"epoch": 0.890625, "grad_norm": 0.11770519614219666, "learning_rate": 6.69036847577983e-05, "loss": 0.0008541655843146145, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00085, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.53, "tokens/trainable": 103675} +{"epoch": 0.89453125, "grad_norm": 0.28230905532836914, "learning_rate": 6.662348872453553e-05, "loss": 0.005725085735321045, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00574, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 104136} +{"epoch": 0.8984375, "grad_norm": 0.38893985748291016, "learning_rate": 6.63428089904618e-05, "loss": 0.008733941242098808, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00877, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 104588} +{"epoch": 0.90234375, "grad_norm": 0.28149521350860596, "learning_rate": 6.60616572358065e-05, "loss": 0.0034227112773805857, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00343, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 104988} +{"epoch": 0.90625, "grad_norm": 0.07757029682397842, "learning_rate": 6.578004516044172e-05, "loss": 0.0009283493272960186, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00093, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 105455} +{"epoch": 0.91015625, "grad_norm": 0.04546886309981346, "learning_rate": 6.549798448339548e-05, "loss": 0.00036609938251785934, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00037, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 105918} +{"epoch": 0.9140625, "grad_norm": 0.5068175792694092, "learning_rate": 6.521548694236384e-05, "loss": 0.003248700173571706, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00325, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.17, "tokens/trainable": 106355} +{"epoch": 0.91796875, "grad_norm": 0.6692119240760803, "learning_rate": 6.493256429322259e-05, "loss": 0.019515860825777054, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01971, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 21.97, "tokens/trainable": 106786} +{"epoch": 0.921875, "grad_norm": 0.12876470386981964, "learning_rate": 6.464922830953799e-05, "loss": 0.001996834995225072, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.002, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.13, "tokens/trainable": 107219} +{"epoch": 0.92578125, "grad_norm": 1.8084551095962524, "learning_rate": 6.436549078207688e-05, "loss": 0.019572274759411812, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.01977, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.52, "tokens/trainable": 107679} +{"epoch": 0.9296875, "grad_norm": 0.33194291591644287, "learning_rate": 6.408136351831592e-05, "loss": 0.003690862562507391, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.0037, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.89, "tokens/trainable": 108119} +{"epoch": 0.93359375, "grad_norm": 0.20632034540176392, "learning_rate": 6.379685834195036e-05, "loss": 0.0031554887536913157, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00316, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 108626} +{"epoch": 0.9375, "grad_norm": 0.30899739265441895, "learning_rate": 6.351198709240186e-05, "loss": 0.007298732176423073, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00733, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.93, "tokens/trainable": 109079} +{"epoch": 0.94140625, "grad_norm": 0.2676924765110016, "learning_rate": 6.32267616243259e-05, "loss": 0.0035320380702614784, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00354, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 109542} +{"epoch": 0.9453125, "grad_norm": 0.3016713261604309, "learning_rate": 6.294119380711849e-05, "loss": 0.004814652726054192, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00483, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.56, "tokens/trainable": 109969} +{"epoch": 0.94921875, "grad_norm": 0.4877925217151642, "learning_rate": 6.265529552442209e-05, "loss": 0.008069408126175404, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.0081, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 110431} +{"epoch": 0.953125, "grad_norm": 0.05085320398211479, "learning_rate": 6.236907867363127e-05, "loss": 0.0006913705728948116, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.00069, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 110877} +{"epoch": 0.95703125, "grad_norm": 0.08469358086585999, "learning_rate": 6.208255516539749e-05, "loss": 0.0012367932358756661, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00124, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 111350} +{"epoch": 0.9609375, "grad_norm": 0.1905834823846817, "learning_rate": 6.179573692313344e-05, "loss": 0.0022413863334804773, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00224, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 111798} +{"epoch": 0.96484375, "grad_norm": 0.5935748815536499, "learning_rate": 6.150863588251694e-05, "loss": 0.007021765224635601, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00705, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.43, "tokens/trainable": 112264} +{"epoch": 0.96875, "grad_norm": 0.02619190141558647, "learning_rate": 6.122126399099419e-05, "loss": 0.00027101111481897533, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00027, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.56, "tokens/trainable": 112772} +{"epoch": 0.97265625, "grad_norm": 0.9872847199440002, "learning_rate": 6.0933633207282615e-05, "loss": 0.019288720563054085, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.01948, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 113278} +{"epoch": 0.9765625, "grad_norm": 0.20942139625549316, "learning_rate": 6.064575550087316e-05, "loss": 0.0029158571269363165, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00292, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.47, "tokens/trainable": 113716} +{"epoch": 0.98046875, "grad_norm": 0.3441846966743469, "learning_rate": 6.0357642851532245e-05, "loss": 0.00659321341663599, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00661, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 114207} +{"epoch": 0.984375, "grad_norm": 2.232477903366089, "learning_rate": 6.0069307248803294e-05, "loss": 0.01673574559390545, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01688, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.72, "tokens/trainable": 114628} +{"epoch": 0.98828125, "grad_norm": 0.6485549807548523, "learning_rate": 5.9780760691507635e-05, "loss": 0.0035901619121432304, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.0036, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.43, "tokens/trainable": 115060} +{"epoch": 0.9921875, "grad_norm": 0.3123009502887726, "learning_rate": 5.9492015187245334e-05, "loss": 0.00209526508115232, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0021, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 115514} +{"epoch": 0.99609375, "grad_norm": 0.5158595442771912, "learning_rate": 5.920308275189541e-05, "loss": 0.020818334072828293, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.02104, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 115915} +{"epoch": 1.0, "grad_norm": 0.26307472586631775, "learning_rate": 5.8913975409115874e-05, "loss": 0.013858775608241558, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01396, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.05, "tokens/trainable": 116378} +{"epoch": 1.00390625, "grad_norm": 0.06802220642566681, "learning_rate": 5.8624705189843395e-05, "loss": 0.0005394761101342738, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00054, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.29, "tokens/trainable": 116865} +{"epoch": 1.0078125, "grad_norm": 0.3138531744480133, "learning_rate": 5.833528413179249e-05, "loss": 0.0019243642454966903, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00193, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 117363} +{"epoch": 1.01171875, "grad_norm": 0.4216671288013458, "learning_rate": 5.80457242789548e-05, "loss": 0.007363268174231052, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00739, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.59, "tokens/trainable": 117860} +{"epoch": 1.015625, "grad_norm": 0.2996716797351837, "learning_rate": 5.77560376810977e-05, "loss": 0.0075480458326637745, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00758, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 118324} +{"epoch": 1.01953125, "grad_norm": 0.07763087749481201, "learning_rate": 5.7466236393263005e-05, "loss": 0.0013677604729309678, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.00137, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 118770} +{"epoch": 1.0234375, "grad_norm": 0.3733329772949219, "learning_rate": 5.717633247526522e-05, "loss": 0.01225972082465887, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01234, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 119206} +{"epoch": 1.02734375, "grad_norm": 0.3204863667488098, "learning_rate": 5.688633799118971e-05, "loss": 0.004316036589443684, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00433, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.03, "tokens/trainable": 119610} +{"epoch": 1.03125, "grad_norm": 0.2862546741962433, "learning_rate": 5.659626500889066e-05, "loss": 0.0034847636707127094, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00349, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 120080} +{"epoch": 1.03515625, "grad_norm": 0.28510215878486633, "learning_rate": 5.6306125599488905e-05, "loss": 0.0028700735419988632, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00287, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 120519} +{"epoch": 1.0390625, "grad_norm": 0.3515375852584839, "learning_rate": 5.601593183686955e-05, "loss": 0.009932870045304298, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00998, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 120995} +{"epoch": 1.04296875, "grad_norm": 0.4166232645511627, "learning_rate": 5.572569579717961e-05, "loss": 0.0037125030066818, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00372, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 121459} +{"epoch": 1.046875, "grad_norm": 0.10788220912218094, "learning_rate": 5.543542955832538e-05, "loss": 0.0019090997520834208, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00191, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.49, "tokens/trainable": 121902} +{"epoch": 1.05078125, "grad_norm": 0.36938077211380005, "learning_rate": 5.514514519946986e-05, "loss": 0.013326050713658333, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.01342, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 122399} +{"epoch": 1.0546875, "grad_norm": 0.07141252607107162, "learning_rate": 5.485485480053015e-05, "loss": 0.0022587915882468224, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.00226, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.71, "tokens/trainable": 122804} +{"epoch": 1.05859375, "grad_norm": 0.3943050503730774, "learning_rate": 5.4564570441674645e-05, "loss": 0.003262361977249384, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00327, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 123244} +{"epoch": 1.0625, "grad_norm": 0.2440444976091385, "learning_rate": 5.42743042028204e-05, "loss": 0.012471755966544151, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.01255, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.34, "tokens/trainable": 123683} +{"epoch": 1.06640625, "grad_norm": 0.30987897515296936, "learning_rate": 5.3984068163130464e-05, "loss": 0.007286457344889641, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.00731, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 124153} +{"epoch": 1.0703125, "grad_norm": 0.06003831326961517, "learning_rate": 5.369387440051111e-05, "loss": 0.0013619285309687257, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00136, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 124616} +{"epoch": 1.07421875, "grad_norm": 0.018491484224796295, "learning_rate": 5.340373499110935e-05, "loss": 0.0006006037583574653, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.0006, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.21, "tokens/trainable": 125109} +{"epoch": 1.078125, "grad_norm": 0.01168125681579113, "learning_rate": 5.3113662008810304e-05, "loss": 0.0003967989468947053, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0004, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 125591} +{"epoch": 1.08203125, "grad_norm": 0.16174520552158356, "learning_rate": 5.282366752473479e-05, "loss": 0.0028332641813904047, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00284, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.32, "tokens/trainable": 126065} +{"epoch": 1.0859375, "grad_norm": 0.09742214530706406, "learning_rate": 5.2533763606737005e-05, "loss": 0.00115255790296942, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00115, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.13, "tokens/trainable": 126480} +{"epoch": 1.08984375, "grad_norm": 0.21307778358459473, "learning_rate": 5.224396231890232e-05, "loss": 0.0016797459684312344, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00168, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 126929} +{"epoch": 1.09375, "grad_norm": 0.2103121131658554, "learning_rate": 5.195427572104522e-05, "loss": 0.005129985511302948, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00514, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 127366} +{"epoch": 1.09765625, "grad_norm": 0.2179725617170334, "learning_rate": 5.166471586820751e-05, "loss": 0.0037892567925155163, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.0038, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.48, "tokens/trainable": 127831} +{"epoch": 1.1015625, "grad_norm": 0.3295300304889679, "learning_rate": 5.1375294810156615e-05, "loss": 0.006468499079346657, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00649, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 128278} +{"epoch": 1.10546875, "grad_norm": 0.034830085933208466, "learning_rate": 5.1086024590884144e-05, "loss": 0.0005982629954814911, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0006, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 128727} +{"epoch": 1.109375, "grad_norm": 0.058777131140232086, "learning_rate": 5.079691724810461e-05, "loss": 0.0010968738934025168, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.0011, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.51, "tokens/trainable": 129162} +{"epoch": 1.11328125, "grad_norm": 0.04980301484465599, "learning_rate": 5.0507984812754684e-05, "loss": 0.000288410228677094, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00029, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 129659} +{"epoch": 1.1171875, "grad_norm": 0.1179724633693695, "learning_rate": 5.021923930849237e-05, "loss": 0.0010806054342538118, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.00108, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 130145} +{"epoch": 1.12109375, "grad_norm": 0.056898750364780426, "learning_rate": 4.99306927511967e-05, "loss": 0.0005650818930007517, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00057, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.63, "tokens/trainable": 130585} +{"epoch": 1.125, "grad_norm": 0.932645320892334, "learning_rate": 4.964235714846775e-05, "loss": 0.004466964863240719, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00448, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 131005} +{"epoch": 1.12890625, "grad_norm": 0.0603514239192009, "learning_rate": 4.9354244499126866e-05, "loss": 0.00034417471033520997, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00034, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 29.09, "tokens/trainable": 131471} +{"epoch": 1.1328125, "grad_norm": 0.028352389112114906, "learning_rate": 4.90663667927174e-05, "loss": 0.00018816307419911027, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00019, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.63, "tokens/trainable": 131920} +{"epoch": 1.13671875, "grad_norm": 0.2586069107055664, "learning_rate": 4.877873600900581e-05, "loss": 0.0009185462258756161, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00092, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.63, "tokens/trainable": 132379} +{"epoch": 1.140625, "grad_norm": 0.2990264594554901, "learning_rate": 4.849136411748306e-05, "loss": 0.0029292369727045298, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00293, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 132851} +{"epoch": 1.14453125, "grad_norm": 0.2969539761543274, "learning_rate": 4.8204263076866574e-05, "loss": 0.008878171443939209, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00892, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.39, "tokens/trainable": 133351} +{"epoch": 1.1484375, "grad_norm": 0.003335407469421625, "learning_rate": 4.791744483460251e-05, "loss": 4.5226213842397556e-05, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00005, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 133784} +{"epoch": 1.15234375, "grad_norm": 0.1349416822195053, "learning_rate": 4.7630921326368736e-05, "loss": 0.0007764893234707415, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00078, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.82, "tokens/trainable": 134209} +{"epoch": 1.15625, "grad_norm": 0.0322023406624794, "learning_rate": 4.7344704475577916e-05, "loss": 0.00016611229511909187, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00017, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 30.07, "tokens/trainable": 134645} +{"epoch": 1.16015625, "grad_norm": 1.5098540782928467, "learning_rate": 4.705880619288153e-05, "loss": 0.008598988875746727, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.00864, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.86, "tokens/trainable": 135047} +{"epoch": 1.1640625, "grad_norm": 0.27084267139434814, "learning_rate": 4.677323837567412e-05, "loss": 0.0012031556107103825, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0012, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 135474} +{"epoch": 1.16796875, "grad_norm": 0.016517069190740585, "learning_rate": 4.6488012907598146e-05, "loss": 7.587560685351491e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00008, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.68, "tokens/trainable": 135948} +{"epoch": 1.171875, "grad_norm": 0.2299436330795288, "learning_rate": 4.620314165804964e-05, "loss": 0.002062344690784812, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00206, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.42, "tokens/trainable": 136421} +{"epoch": 1.17578125, "grad_norm": 0.3665059804916382, "learning_rate": 4.591863648168407e-05, "loss": 0.001791703631170094, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00179, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 136886} +{"epoch": 1.1796875, "grad_norm": 0.006042965222150087, "learning_rate": 4.5634509217923135e-05, "loss": 0.00011143341544084251, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00011, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 137321} +{"epoch": 1.18359375, "grad_norm": 0.597837507724762, "learning_rate": 4.535077169046201e-05, "loss": 0.006236453540623188, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00626, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 24.88, "tokens/trainable": 137758} +{"epoch": 1.1875, "grad_norm": 0.7580564618110657, "learning_rate": 4.506743570677743e-05, "loss": 0.028117220848798752, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.02852, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 30.97, "tokens/trainable": 138190} +{"epoch": 1.19140625, "grad_norm": 0.029939016327261925, "learning_rate": 4.478451305763618e-05, "loss": 0.0003610669227782637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00036, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 138650} +{"epoch": 1.1953125, "grad_norm": 0.011628447100520134, "learning_rate": 4.450201551660454e-05, "loss": 9.215858153766021e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00009, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.05, "tokens/trainable": 139082} +{"epoch": 1.19921875, "grad_norm": 0.15230520069599152, "learning_rate": 4.4219954839558276e-05, "loss": 0.0015517660649493337, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00155, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.21, "tokens/trainable": 139523} +{"epoch": 1.203125, "grad_norm": 0.20869530737400055, "learning_rate": 4.393834276419352e-05, "loss": 0.005242598243057728, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00526, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 32.1, "tokens/trainable": 140010} +{"epoch": 1.20703125, "grad_norm": 0.9505382180213928, "learning_rate": 4.36571910095382e-05, "loss": 0.028183212503790855, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.02858, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.24, "tokens/trainable": 140454} +{"epoch": 1.2109375, "grad_norm": 0.5465441942214966, "learning_rate": 4.337651127546448e-05, "loss": 0.013661866076290607, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.01376, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 140905} +{"epoch": 1.21484375, "grad_norm": 0.05790548771619797, "learning_rate": 4.3096315242201736e-05, "loss": 0.0006652080919593573, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.00067, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 141339} +{"epoch": 1.21875, "grad_norm": 0.17393185198307037, "learning_rate": 4.2816614569850635e-05, "loss": 0.0007501068757846951, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00075, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 141778} +{"epoch": 1.22265625, "grad_norm": 0.448273628950119, "learning_rate": 4.2537420897897864e-05, "loss": 0.02007385343313217, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.02028, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 142255} +{"epoch": 1.2265625, "grad_norm": 0.05531203746795654, "learning_rate": 4.225874584473174e-05, "loss": 0.0010162900434806943, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00102, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.76, "tokens/trainable": 142693} +{"epoch": 1.23046875, "grad_norm": 0.28432896733283997, "learning_rate": 4.19806010071587e-05, "loss": 0.0038717619609087706, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00388, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.08, "tokens/trainable": 143157} +{"epoch": 1.234375, "grad_norm": 0.41314786672592163, "learning_rate": 4.170299795992081e-05, "loss": 0.008719892241060734, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00876, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 143603} +{"epoch": 1.23828125, "grad_norm": 0.12087827920913696, "learning_rate": 4.142594825521398e-05, "loss": 0.002730270614847541, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00273, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 144084} +{"epoch": 1.2421875, "grad_norm": 0.08728651702404022, "learning_rate": 4.114946342220728e-05, "loss": 0.0015566409565508366, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00156, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 144532} +{"epoch": 1.24609375, "grad_norm": 0.4511994421482086, "learning_rate": 4.087355496656321e-05, "loss": 0.005300460383296013, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00531, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 145004} +{"epoch": 1.25, "grad_norm": 0.12575769424438477, "learning_rate": 4.05982343699588e-05, "loss": 0.0016395744169130921, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00164, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 145498} +{"epoch": 1.25390625, "grad_norm": 0.21720358729362488, "learning_rate": 4.0323513089607876e-05, "loss": 0.002911246381700039, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00292, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 23.95, "tokens/trainable": 145965} +{"epoch": 1.2578125, "grad_norm": 0.21761198341846466, "learning_rate": 4.004940255778431e-05, "loss": 0.0024054660461843014, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00241, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.72, "tokens/trainable": 146409} +{"epoch": 1.26171875, "grad_norm": 0.6883000731468201, "learning_rate": 3.977591418134619e-05, "loss": 0.008925714530050755, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00897, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 146849} +{"epoch": 1.265625, "grad_norm": 0.1587570458650589, "learning_rate": 3.95030593412612e-05, "loss": 0.002462641801685095, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00247, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 147272} +{"epoch": 1.26953125, "grad_norm": 0.024494513869285583, "learning_rate": 3.923084939213296e-05, "loss": 0.0005384684191085398, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00054, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 147694} +{"epoch": 1.2734375, "grad_norm": 0.366073340177536, "learning_rate": 3.895929566172861e-05, "loss": 0.017883365973830223, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01804, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 25.01, "tokens/trainable": 148158} +{"epoch": 1.27734375, "grad_norm": 0.2734401226043701, "learning_rate": 3.868840945050728e-05, "loss": 0.0014041891554370522, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.00141, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.32, "tokens/trainable": 148590} +{"epoch": 1.28125, "grad_norm": 0.18794989585876465, "learning_rate": 3.841820203114995e-05, "loss": 0.004809961188584566, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00482, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.85, "tokens/trainable": 149037} +{"epoch": 1.28515625, "grad_norm": 0.06745494902133942, "learning_rate": 3.814868464809027e-05, "loss": 0.0007819278398528695, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00078, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 149439} +{"epoch": 1.2890625, "grad_norm": 0.31079041957855225, "learning_rate": 3.787986851704667e-05, "loss": 0.0040690964087843895, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00408, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.61, "tokens/trainable": 149869} +{"epoch": 1.29296875, "grad_norm": 0.11979186534881592, "learning_rate": 3.7611764824555654e-05, "loss": 0.002039270242676139, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00204, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.7, "tokens/trainable": 150343} +{"epoch": 1.296875, "grad_norm": 0.0391436330974102, "learning_rate": 3.734438472750619e-05, "loss": 0.0010497216135263443, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.00105, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 23.04, "tokens/trainable": 150814} +{"epoch": 1.30078125, "grad_norm": 0.016679126769304276, "learning_rate": 3.707773935267552e-05, "loss": 0.00034638209035620093, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00035, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.81, "tokens/trainable": 151300} +{"epoch": 1.3046875, "grad_norm": 0.09574971348047256, "learning_rate": 3.68118397962661e-05, "loss": 0.0011816725600510836, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00118, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.16, "tokens/trainable": 151708} +{"epoch": 1.30859375, "grad_norm": 0.06749123334884644, "learning_rate": 3.654669712344384e-05, "loss": 0.0009392376523464918, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00094, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 152164} +{"epoch": 1.3125, "grad_norm": 0.23438754677772522, "learning_rate": 3.628232236787763e-05, "loss": 0.0018706817645579576, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00187, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.52, "tokens/trainable": 152603} +{"epoch": 1.31640625, "grad_norm": 0.030918734148144722, "learning_rate": 3.6018726531280144e-05, "loss": 0.00032267096685245633, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00032, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.11, "tokens/trainable": 153094} +{"epoch": 1.3203125, "grad_norm": 0.7222067713737488, "learning_rate": 3.575592058295017e-05, "loss": 0.005238580983132124, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.00525, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 153573} +{"epoch": 1.32421875, "grad_norm": 0.010666495189070702, "learning_rate": 3.549391545931585e-05, "loss": 0.00023126497399061918, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00023, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 154033} +{"epoch": 1.328125, "grad_norm": 0.024763235822319984, "learning_rate": 3.5232722063479914e-05, "loss": 0.00016378167492803186, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00016, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 24.05, "tokens/trainable": 154482} +{"epoch": 1.33203125, "grad_norm": 0.1687019169330597, "learning_rate": 3.49723512647657e-05, "loss": 0.0014253004919737577, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00143, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.61, "tokens/trainable": 154982} +{"epoch": 1.3359375, "grad_norm": 0.02495335228741169, "learning_rate": 3.471281389826491e-05, "loss": 0.0002516527893021703, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00025, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.82, "tokens/trainable": 155387} +{"epoch": 1.33984375, "grad_norm": 0.7915899753570557, "learning_rate": 3.4454120764386764e-05, "loss": 0.012272713705897331, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.01235, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.15, "tokens/trainable": 155842} +{"epoch": 1.34375, "grad_norm": 0.2964228093624115, "learning_rate": 3.4196282628408526e-05, "loss": 0.0023426716215908527, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00235, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 156287} +{"epoch": 1.34765625, "grad_norm": 0.5572789311408997, "learning_rate": 3.3939310220027456e-05, "loss": 0.0059123653918504715, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00593, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.68, "tokens/trainable": 156782} +{"epoch": 1.3515625, "grad_norm": 0.20401757955551147, "learning_rate": 3.3683214232914404e-05, "loss": 0.001341738272458315, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00134, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 157255} +{"epoch": 1.35546875, "grad_norm": 0.012613953091204166, "learning_rate": 3.342800532426873e-05, "loss": 0.00020807070541195571, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00021, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.39, "tokens/trainable": 157721} +{"epoch": 1.359375, "grad_norm": 0.037366077303886414, "learning_rate": 3.317369411437484e-05, "loss": 0.00033415568759664893, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00033, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 158189} +{"epoch": 1.36328125, "grad_norm": 0.09746008366346359, "learning_rate": 3.292029118616024e-05, "loss": 0.0006656049517914653, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00067, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 158612} +{"epoch": 1.3671875, "grad_norm": 0.20046578347682953, "learning_rate": 3.266780708475511e-05, "loss": 0.0019085367675870657, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00191, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.9, "tokens/trainable": 159090} +{"epoch": 1.37109375, "grad_norm": 0.3691701591014862, "learning_rate": 3.241625231705354e-05, "loss": 0.0031896380241960287, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00319, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.87, "tokens/trainable": 159549} +{"epoch": 1.375, "grad_norm": 0.01767587475478649, "learning_rate": 3.216563735127618e-05, "loss": 0.0003243396640755236, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00032, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 160026} +{"epoch": 1.37890625, "grad_norm": 0.019548693671822548, "learning_rate": 3.191597261653475e-05, "loss": 0.0003194265882484615, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00032, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 160487} +{"epoch": 1.3828125, "grad_norm": 0.17453992366790771, "learning_rate": 3.166726850239794e-05, "loss": 0.00209711492061615, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0021, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 160935} +{"epoch": 1.38671875, "grad_norm": 0.03717486560344696, "learning_rate": 3.141953535845912e-05, "loss": 0.00033541605807840824, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00034, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.3, "tokens/trainable": 161385} +{"epoch": 1.390625, "grad_norm": 1.8252365589141846, "learning_rate": 3.11727834939056e-05, "loss": 0.0025314227677881718, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00253, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 161849} +{"epoch": 1.39453125, "grad_norm": 0.17204435169696808, "learning_rate": 3.092702317708967e-05, "loss": 0.01021526101976633, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.01027, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 162306} +{"epoch": 1.3984375, "grad_norm": 0.023266257718205452, "learning_rate": 3.0682264635101276e-05, "loss": 0.0003625139652285725, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00036, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.16, "tokens/trainable": 162759} +{"epoch": 1.40234375, "grad_norm": 0.33819133043289185, "learning_rate": 3.0438518053342407e-05, "loss": 0.005052329506725073, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00507, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.07, "tokens/trainable": 163230} +{"epoch": 1.40625, "grad_norm": 0.23095010221004486, "learning_rate": 3.0195793575103266e-05, "loss": 0.0028241085819900036, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00283, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.21, "tokens/trainable": 163694} +{"epoch": 1.41015625, "grad_norm": 0.01157522015273571, "learning_rate": 2.9954101301140146e-05, "loss": 0.0002799902867991477, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00028, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.5, "tokens/trainable": 164144} +{"epoch": 1.4140625, "grad_norm": 0.4266413152217865, "learning_rate": 2.9713451289255123e-05, "loss": 0.004659952130168676, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00467, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 164585} +{"epoch": 1.41796875, "grad_norm": 0.012600954622030258, "learning_rate": 2.9473853553877484e-05, "loss": 0.00022600065858568996, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00023, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 165059} +{"epoch": 1.421875, "grad_norm": 0.15357749164104462, "learning_rate": 2.9235318065647e-05, "loss": 0.0012445549946278334, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00125, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 165519} +{"epoch": 1.42578125, "grad_norm": 0.08849358558654785, "learning_rate": 2.8997854750998964e-05, "loss": 0.0007413022685796022, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00074, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.09, "tokens/trainable": 165984} +{"epoch": 1.4296875, "grad_norm": 0.3209187090396881, "learning_rate": 2.8761473491751258e-05, "loss": 0.001892200903967023, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00189, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.85, "tokens/trainable": 166419} +{"epoch": 1.43359375, "grad_norm": 0.42001378536224365, "learning_rate": 2.8526184124692883e-05, "loss": 0.0037618624046444893, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00377, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.33, "tokens/trainable": 166857} +{"epoch": 1.4375, "grad_norm": 0.5370978713035583, "learning_rate": 2.829199644117484e-05, "loss": 0.00639633322134614, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.00642, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 167267} +{"epoch": 1.44140625, "grad_norm": 0.8441697955131531, "learning_rate": 2.8058920186702553e-05, "loss": 0.012277328409254551, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01235, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 167740} +{"epoch": 1.4453125, "grad_norm": 0.10976596921682358, "learning_rate": 2.782696506053033e-05, "loss": 0.0007580065866932273, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00076, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.26, "tokens/trainable": 168172} +{"epoch": 1.44921875, "grad_norm": 0.5442355275154114, "learning_rate": 2.7596140715257824e-05, "loss": 0.00834126491099596, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00838, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 168637} +{"epoch": 1.453125, "grad_norm": 0.02564423717558384, "learning_rate": 2.7366456756428184e-05, "loss": 0.00031878019217401743, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00032, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 169094} +{"epoch": 1.45703125, "grad_norm": 0.009609510190784931, "learning_rate": 2.7137922742128486e-05, "loss": 0.00018030780483968556, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00018, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 169593} +{"epoch": 1.4609375, "grad_norm": 0.007567352149635553, "learning_rate": 2.691054818259188e-05, "loss": 0.00015315975178964436, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00015, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 170066} +{"epoch": 1.46484375, "grad_norm": 0.17241016030311584, "learning_rate": 2.6684342539801933e-05, "loss": 0.001255715498700738, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00126, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.65, "tokens/trainable": 170518} +{"epoch": 1.46875, "grad_norm": 0.057769015431404114, "learning_rate": 2.645931522709877e-05, "loss": 0.0004923196975141764, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00049, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 28.9, "tokens/trainable": 170998} +{"epoch": 1.47265625, "grad_norm": 0.03967905417084694, "learning_rate": 2.6235475608787365e-05, "loss": 0.00022106363030616194, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00022, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 171459} +{"epoch": 1.4765625, "grad_norm": 0.3784139156341553, "learning_rate": 2.6012832999747916e-05, "loss": 0.008559835143387318, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.0086, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.27, "tokens/trainable": 171905} +{"epoch": 1.48046875, "grad_norm": 0.2542298436164856, "learning_rate": 2.579139666504821e-05, "loss": 0.01606750302016735, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.0162, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 22.01, "tokens/trainable": 172351} +{"epoch": 1.484375, "grad_norm": 0.04241756722331047, "learning_rate": 2.557117581955798e-05, "loss": 0.00039802922401577234, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.0004, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 172817} +{"epoch": 1.48828125, "grad_norm": 0.20263409614562988, "learning_rate": 2.5352179627565532e-05, "loss": 0.0025753688532859087, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00258, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 173258} +{"epoch": 1.4921875, "grad_norm": 0.026950566098093987, "learning_rate": 2.5134417202396277e-05, "loss": 0.0004676379030570388, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00047, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 173665} +{"epoch": 1.49609375, "grad_norm": 0.4596717357635498, "learning_rate": 2.491789760603361e-05, "loss": 0.006490319035947323, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00651, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.1, "tokens/trainable": 174095} +{"epoch": 1.5, "grad_norm": 0.13438630104064941, "learning_rate": 2.4702629848741764e-05, "loss": 0.0009902040474116802, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00099, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 174550} +{"epoch": 1.50390625, "grad_norm": 0.015780584886670113, "learning_rate": 2.4488622888690785e-05, "loss": 0.00023736112052574754, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.00024, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 174990} +{"epoch": 1.5078125, "grad_norm": 0.016738293692469597, "learning_rate": 2.427588563158384e-05, "loss": 0.00018222736252937466, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00018, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.03, "tokens/trainable": 175476} +{"epoch": 1.51171875, "grad_norm": 0.39864933490753174, "learning_rate": 2.406442693028651e-05, "loss": 0.003185129025951028, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00319, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 175897} +{"epoch": 1.515625, "grad_norm": 0.37510743737220764, "learning_rate": 2.3854255584458547e-05, "loss": 0.0004094602190889418, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00041, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.49, "tokens/trainable": 176324} +{"epoch": 1.51953125, "grad_norm": 0.030201952904462814, "learning_rate": 2.3645380340187508e-05, "loss": 0.00031422392930835485, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00031, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 176805} +{"epoch": 1.5234375, "grad_norm": 0.26535165309906006, "learning_rate": 2.3437809889624914e-05, "loss": 0.002442945260554552, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00245, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 29.05, "tokens/trainable": 177256} +{"epoch": 1.52734375, "grad_norm": 0.27177003026008606, "learning_rate": 2.3231552870624487e-05, "loss": 0.0027609739918261766, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00276, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 177682} +{"epoch": 1.53125, "grad_norm": 0.012493660673499107, "learning_rate": 2.3026617866382657e-05, "loss": 0.00013925277744419873, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00014, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.73, "tokens/trainable": 178179} +{"epoch": 1.53515625, "grad_norm": 0.32816752791404724, "learning_rate": 2.2823013405081507e-05, "loss": 0.002190191764384508, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00219, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 178676} +{"epoch": 1.5390625, "grad_norm": 0.017341002821922302, "learning_rate": 2.2620747959533722e-05, "loss": 0.00015673317830078304, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00016, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.52, "tokens/trainable": 179173} +{"epoch": 1.54296875, "grad_norm": 0.01690303161740303, "learning_rate": 2.2419829946830123e-05, "loss": 0.00017251449753530324, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00017, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 179624} +{"epoch": 1.546875, "grad_norm": 0.30249637365341187, "learning_rate": 2.2220267727989325e-05, "loss": 0.007945846766233444, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00798, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 180078} +{"epoch": 1.55078125, "grad_norm": 0.07539468258619308, "learning_rate": 2.202206960760984e-05, "loss": 0.0010492957662791014, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00105, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.59, "tokens/trainable": 180585} +{"epoch": 1.5546875, "grad_norm": 0.01915985904633999, "learning_rate": 2.182524383352446e-05, "loss": 0.00041690125362947583, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00042, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 181039} +{"epoch": 1.55859375, "grad_norm": 0.2513546049594879, "learning_rate": 2.1629798596457056e-05, "loss": 0.011237140744924545, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.0113, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.6, "tokens/trainable": 181489} +{"epoch": 1.5625, "grad_norm": 0.009297998622059822, "learning_rate": 2.1435742029681725e-05, "loss": 0.0001503608946222812, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00015, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 181906} +{"epoch": 1.56640625, "grad_norm": 0.00533163221552968, "learning_rate": 2.124308220868431e-05, "loss": 0.00012705489643849432, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00013, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.2, "tokens/trainable": 182361} +{"epoch": 1.5703125, "grad_norm": 0.06822330504655838, "learning_rate": 2.105182715082638e-05, "loss": 0.0014628346543759108, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00146, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 182838} +{"epoch": 1.57421875, "grad_norm": 0.04661620408296585, "learning_rate": 2.0861984815011552e-05, "loss": 0.0005374523461796343, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00054, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.53, "tokens/trainable": 183269} +{"epoch": 1.578125, "grad_norm": 0.03739500790834427, "learning_rate": 2.0673563101354323e-05, "loss": 0.0002946380409412086, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00029, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.64, "tokens/trainable": 183699} +{"epoch": 1.58203125, "grad_norm": 0.4552408456802368, "learning_rate": 2.0486569850851317e-05, "loss": 0.003118939930573106, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00312, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.83, "tokens/trainable": 184162} +{"epoch": 1.5859375, "grad_norm": 0.030456990003585815, "learning_rate": 2.0301012845054956e-05, "loss": 0.00038440380012616515, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00038, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.48, "tokens/trainable": 184576} +{"epoch": 1.58984375, "grad_norm": 0.03526407852768898, "learning_rate": 2.011689980574966e-05, "loss": 0.0004551718011498451, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00046, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.78, "tokens/trainable": 184995} +{"epoch": 1.59375, "grad_norm": 0.2701110243797302, "learning_rate": 1.993423839463052e-05, "loss": 0.0039007999002933502, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00391, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.76, "tokens/trainable": 185452} +{"epoch": 1.59765625, "grad_norm": 0.12068334966897964, "learning_rate": 1.975303621298445e-05, "loss": 0.0010728038614615798, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00107, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 185904} +{"epoch": 1.6015625, "grad_norm": 0.03437642380595207, "learning_rate": 1.957330080137385e-05, "loss": 0.0003986497758887708, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.0004, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.96, "tokens/trainable": 186355} +{"epoch": 1.60546875, "grad_norm": 0.014549183659255505, "learning_rate": 1.9395039639322864e-05, "loss": 0.0002389691217103973, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00024, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.34, "tokens/trainable": 186831} +{"epoch": 1.609375, "grad_norm": 0.008174730464816093, "learning_rate": 1.9218260145006073e-05, "loss": 9.163570211967453e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00009, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.97, "tokens/trainable": 187317} +{"epoch": 1.61328125, "grad_norm": 0.014930643141269684, "learning_rate": 1.904296967493982e-05, "loss": 0.0002174567780457437, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00022, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.65, "tokens/trainable": 187773} +{"epoch": 1.6171875, "grad_norm": 0.08347151428461075, "learning_rate": 1.8869175523676064e-05, "loss": 0.001127884490415454, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00113, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 188268} +{"epoch": 1.62109375, "grad_norm": 0.03482385724782944, "learning_rate": 1.869688492349885e-05, "loss": 0.0005082964198663831, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00051, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 188720} +{"epoch": 1.625, "grad_norm": 0.012837070971727371, "learning_rate": 1.85261050441233e-05, "loss": 0.0002132733352482319, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00021, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 189177} +{"epoch": 1.62890625, "grad_norm": 0.4167100787162781, "learning_rate": 1.8356842992397304e-05, "loss": 0.0150532778352499, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01517, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 27.02, "tokens/trainable": 189617} +{"epoch": 1.6328125, "grad_norm": 0.038745976984500885, "learning_rate": 1.8189105812005714e-05, "loss": 0.00051952461944893, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00052, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.31, "tokens/trainable": 190028} +{"epoch": 1.63671875, "grad_norm": 0.032912980765104294, "learning_rate": 1.802290048317732e-05, "loss": 0.0005818502977490425, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00058, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.35, "tokens/trainable": 190445} +{"epoch": 1.640625, "grad_norm": 0.005901315249502659, "learning_rate": 1.785823392239424e-05, "loss": 0.00011268975504208356, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00011, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 190886} +{"epoch": 1.64453125, "grad_norm": 0.05779954046010971, "learning_rate": 1.7695112982104225e-05, "loss": 0.0008973665535449982, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.0009, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.84, "tokens/trainable": 191325} +{"epoch": 1.6484375, "grad_norm": 0.012219991534948349, "learning_rate": 1.7533544450435433e-05, "loss": 0.0002053958596661687, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00021, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 30.0, "tokens/trainable": 191758} +{"epoch": 1.65234375, "grad_norm": 0.006649721879512072, "learning_rate": 1.7373535050913946e-05, "loss": 0.00014462567924056202, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00014, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.81, "tokens/trainable": 192195} +{"epoch": 1.65625, "grad_norm": 0.056614816188812256, "learning_rate": 1.721509144218405e-05, "loss": 0.0006590132834389806, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.00066, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.78, "tokens/trainable": 192625} +{"epoch": 1.66015625, "grad_norm": 0.007133859675377607, "learning_rate": 1.705822021773101e-05, "loss": 0.00017400582146365196, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00017, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.92, "tokens/trainable": 193111} +{"epoch": 1.6640625, "grad_norm": 0.2669585943222046, "learning_rate": 1.69029279056068e-05, "loss": 0.010343372821807861, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.0104, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 193551} +{"epoch": 1.66796875, "grad_norm": 0.18884910643100739, "learning_rate": 1.6749220968158415e-05, "loss": 0.0020765752997249365, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00208, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 29.06, "tokens/trainable": 194057} +{"epoch": 1.671875, "grad_norm": 0.015237867832183838, "learning_rate": 1.659710580175893e-05, "loss": 0.0002654343261383474, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00027, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 32.09, "tokens/trainable": 194556} +{"epoch": 1.67578125, "grad_norm": 0.16667833924293518, "learning_rate": 1.644658873654133e-05, "loss": 0.001206223969347775, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00121, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.37, "tokens/trainable": 194993} +{"epoch": 1.6796875, "grad_norm": 0.022254744544625282, "learning_rate": 1.629767603613508e-05, "loss": 0.00014762012870050967, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00015, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.7, "tokens/trainable": 195393} +{"epoch": 1.68359375, "grad_norm": 0.34379664063453674, "learning_rate": 1.615037389740547e-05, "loss": 0.004957697354257107, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00497, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.9, "tokens/trainable": 195837} +{"epoch": 1.6875, "grad_norm": 0.035976190119981766, "learning_rate": 1.600468845019576e-05, "loss": 0.00023518870875705034, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00024, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.75, "tokens/trainable": 196317} +{"epoch": 1.69140625, "grad_norm": 0.009366419166326523, "learning_rate": 1.5860625757072092e-05, "loss": 0.00011980260751442984, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00012, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.9, "tokens/trainable": 196765} +{"epoch": 1.6953125, "grad_norm": 0.0062650395557284355, "learning_rate": 1.571819181307116e-05, "loss": 8.240649185609072e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00008, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 197237} +{"epoch": 1.69921875, "grad_norm": 0.07359833270311356, "learning_rate": 1.557739254545075e-05, "loss": 0.0006461554439738393, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00065, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.65, "tokens/trainable": 197709} +{"epoch": 1.703125, "grad_norm": 0.0030547359492629766, "learning_rate": 1.543823381344311e-05, "loss": 7.793247641529888e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00008, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 31.06, "tokens/trainable": 198184} +{"epoch": 1.70703125, "grad_norm": 0.05375076085329056, "learning_rate": 1.5300721408011114e-05, "loss": 0.0008124231244437397, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00081, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.7, "tokens/trainable": 198641} +{"epoch": 1.7109375, "grad_norm": 0.013841675594449043, "learning_rate": 1.5164861051607254e-05, "loss": 0.00021221095812506974, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00021, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 199049} +{"epoch": 1.71484375, "grad_norm": 0.03508402407169342, "learning_rate": 1.5030658397935521e-05, "loss": 0.0003119028697255999, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00031, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.55, "tokens/trainable": 199518} +{"epoch": 1.71875, "grad_norm": 0.16081805527210236, "learning_rate": 1.4898119031716104e-05, "loss": 0.0013412130065262318, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00134, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 199976} +{"epoch": 1.72265625, "grad_norm": 0.24527131021022797, "learning_rate": 1.476724846845306e-05, "loss": 0.002490152372047305, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00249, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.3, "tokens/trainable": 200453} +{"epoch": 1.7265625, "grad_norm": 0.07340318709611893, "learning_rate": 1.463805215420471e-05, "loss": 0.0004555560299195349, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00046, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 200923} +{"epoch": 1.73046875, "grad_norm": 0.002229720586910844, "learning_rate": 1.451053546535705e-05, "loss": 7.022848876658827e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00007, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 201379} +{"epoch": 1.734375, "grad_norm": 0.034313466399908066, "learning_rate": 1.438470370840001e-05, "loss": 0.000257513573160395, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00026, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 201851} +{"epoch": 1.73828125, "grad_norm": 0.013477550819516182, "learning_rate": 1.4260562119706606e-05, "loss": 0.000250689045060426, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00025, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.17, "tokens/trainable": 202325} +{"epoch": 1.7421875, "grad_norm": 0.01865416392683983, "learning_rate": 1.413811586531508e-05, "loss": 0.00028756045503541827, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00029, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.6, "tokens/trainable": 202766} +{"epoch": 1.74609375, "grad_norm": 0.018986664712429047, "learning_rate": 1.4017370040713884e-05, "loss": 0.00035321118775755167, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00035, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 203215} +{"epoch": 1.75, "grad_norm": 0.12586244940757751, "learning_rate": 1.3898329670629645e-05, "loss": 0.0008539830450899899, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00085, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.98, "tokens/trainable": 203660} +{"epoch": 1.75390625, "grad_norm": 0.00492165656760335, "learning_rate": 1.3780999708818058e-05, "loss": 0.00010040247434517369, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.0001, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 204110} +{"epoch": 1.7578125, "grad_norm": 0.0045965020544826984, "learning_rate": 1.3665385037857758e-05, "loss": 8.087164314929396e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00008, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.68, "tokens/trainable": 204562} +{"epoch": 1.76171875, "grad_norm": 0.08030141890048981, "learning_rate": 1.3551490468947126e-05, "loss": 0.0006682703970000148, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00067, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.51, "tokens/trainable": 205008} +{"epoch": 1.765625, "grad_norm": 0.11196807771921158, "learning_rate": 1.3439320741704075e-05, "loss": 0.000993457157164812, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00099, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 205432} +{"epoch": 1.76953125, "grad_norm": 0.020274560898542404, "learning_rate": 1.3328880523968808e-05, "loss": 0.00020066418801434338, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.0002, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 205895} +{"epoch": 1.7734375, "grad_norm": 0.00510428287088871, "learning_rate": 1.3220174411609587e-05, "loss": 0.000127111779875122, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00013, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.64, "tokens/trainable": 206334} +{"epoch": 1.77734375, "grad_norm": 0.1673465073108673, "learning_rate": 1.3113206928331471e-05, "loss": 0.0018764800624921918, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00188, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.6, "tokens/trainable": 206796} +{"epoch": 1.78125, "grad_norm": 0.06995003670454025, "learning_rate": 1.300798252548806e-05, "loss": 0.00046209723223000765, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00046, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.32, "tokens/trainable": 207248} +{"epoch": 1.78515625, "grad_norm": 0.3524205684661865, "learning_rate": 1.2904505581896265e-05, "loss": 0.0075547173619270325, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00758, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.3, "tokens/trainable": 207721} +{"epoch": 1.7890625, "grad_norm": 0.052947044372558594, "learning_rate": 1.2802780403654082e-05, "loss": 0.0006048440118320286, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00061, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.57, "tokens/trainable": 208168} +{"epoch": 1.79296875, "grad_norm": 0.01054803840816021, "learning_rate": 1.2702811223961408e-05, "loss": 0.00016390837845392525, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00016, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 208629} +{"epoch": 1.796875, "grad_norm": 0.009634353220462799, "learning_rate": 1.2604602202943861e-05, "loss": 8.290822006529197e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00008, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.23, "tokens/trainable": 209073} +{"epoch": 1.80078125, "grad_norm": 0.27039989829063416, "learning_rate": 1.2508157427479686e-05, "loss": 0.0032360763289034367, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00324, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.27, "tokens/trainable": 209520} +{"epoch": 1.8046875, "grad_norm": 0.007729407399892807, "learning_rate": 1.2413480911029655e-05, "loss": 0.0001473601150792092, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00015, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.91, "tokens/trainable": 209958} +{"epoch": 1.80859375, "grad_norm": 0.0680050477385521, "learning_rate": 1.2320576593470082e-05, "loss": 0.0005393248284235597, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00054, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.94, "tokens/trainable": 210385} +{"epoch": 1.8125, "grad_norm": 0.31426164507865906, "learning_rate": 1.2229448340928828e-05, "loss": 0.0054423613473773, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00546, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 210893} +{"epoch": 1.81640625, "grad_norm": 0.004545530769973993, "learning_rate": 1.2140099945624458e-05, "loss": 0.000122253957670182, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00012, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 22.98, "tokens/trainable": 211350} +{"epoch": 1.8203125, "grad_norm": 0.40698838233947754, "learning_rate": 1.205253512570841e-05, "loss": 0.00484098494052887, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00485, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 211798} +{"epoch": 1.82421875, "grad_norm": 0.016746539622545242, "learning_rate": 1.1966757525110255e-05, "loss": 0.00024277633929159492, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00024, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.72, "tokens/trainable": 212249} +{"epoch": 1.828125, "grad_norm": 0.3041616380214691, "learning_rate": 1.1882770713386095e-05, "loss": 0.0015779630048200488, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00158, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 26.01, "tokens/trainable": 212724} +{"epoch": 1.83203125, "grad_norm": 0.006186901591718197, "learning_rate": 1.180057818556998e-05, "loss": 8.15401435829699e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00008, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.66, "tokens/trainable": 213199} +{"epoch": 1.8359375, "grad_norm": 0.019262967631220818, "learning_rate": 1.1720183362028494e-05, "loss": 0.0001529709406895563, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00015, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.85, "tokens/trainable": 213664} +{"epoch": 1.83984375, "grad_norm": 0.00559092266485095, "learning_rate": 1.1641589588318387e-05, "loss": 9.830405178945512e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0001, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.29, "tokens/trainable": 214109} +{"epoch": 1.84375, "grad_norm": 0.0031211248133331537, "learning_rate": 1.1564800135047418e-05, "loss": 4.279188215150498e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00004, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 214597} +{"epoch": 1.84765625, "grad_norm": 0.012266561388969421, "learning_rate": 1.148981819773816e-05, "loss": 0.00017947432934306562, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00018, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 215014} +{"epoch": 1.8515625, "grad_norm": 0.013465085066854954, "learning_rate": 1.1416646896695086e-05, "loss": 8.163638267433271e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00008, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.49, "tokens/trainable": 215466} +{"epoch": 1.85546875, "grad_norm": 0.08602235466241837, "learning_rate": 1.1345289276874717e-05, "loss": 0.001125336973927915, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00113, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.41, "tokens/trainable": 215937} +{"epoch": 1.859375, "grad_norm": 0.002663360210135579, "learning_rate": 1.1275748307758873e-05, "loss": 5.7762506912695244e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00006, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.69, "tokens/trainable": 216405} +{"epoch": 1.86328125, "grad_norm": 0.12924547493457794, "learning_rate": 1.1208026883231147e-05, "loss": 0.0007495130994357169, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00075, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 216820} +{"epoch": 1.8671875, "grad_norm": 0.013708481565117836, "learning_rate": 1.1142127821456433e-05, "loss": 0.00023046473506838083, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00023, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 217270} +{"epoch": 1.87109375, "grad_norm": 0.050574541091918945, "learning_rate": 1.1078053864763674e-05, "loss": 0.00044994568452239037, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00045, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.77, "tokens/trainable": 217722} +{"epoch": 1.875, "grad_norm": 0.06808391958475113, "learning_rate": 1.1015807679531756e-05, "loss": 0.0007722540758550167, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00077, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.19, "tokens/trainable": 218170} +{"epoch": 1.87890625, "grad_norm": 0.05179626867175102, "learning_rate": 1.0955391856078528e-05, "loss": 0.00037320193951018155, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00037, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 218644} +{"epoch": 1.8828125, "grad_norm": 0.4499901235103607, "learning_rate": 1.0896808908553007e-05, "loss": 0.003506165463477373, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00351, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 219092} +{"epoch": 1.88671875, "grad_norm": 0.7384281158447266, "learning_rate": 1.0840061274830763e-05, "loss": 0.007391605991870165, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00742, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 219558} +{"epoch": 1.890625, "grad_norm": 0.0077129085548222065, "learning_rate": 1.0785151316412473e-05, "loss": 0.00012664753012359142, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00013, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 220007} +{"epoch": 1.89453125, "grad_norm": 0.0032921324018388987, "learning_rate": 1.0732081318325639e-05, "loss": 6.325534195639193e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00006, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 220505} +{"epoch": 1.8984375, "grad_norm": 0.12393735349178314, "learning_rate": 1.0680853489029501e-05, "loss": 0.0005243526538833976, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00052, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.83, "tokens/trainable": 220936} +{"epoch": 1.90234375, "grad_norm": 0.07560538500547409, "learning_rate": 1.0631469960323152e-05, "loss": 0.0007506046094931662, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00075, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.53, "tokens/trainable": 221372} +{"epoch": 1.90625, "grad_norm": 0.06888294965028763, "learning_rate": 1.0583932787256783e-05, "loss": 0.0007119372021406889, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00071, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 221792} +{"epoch": 1.91015625, "grad_norm": 0.004689331166446209, "learning_rate": 1.0538243948046206e-05, "loss": 0.00010394290438853204, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.0001, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 222238} +{"epoch": 1.9140625, "grad_norm": 0.009604277089238167, "learning_rate": 1.0494405343990523e-05, "loss": 0.00013647889136336744, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00014, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 222682} +{"epoch": 1.91796875, "grad_norm": 0.01260096114128828, "learning_rate": 1.0452418799392985e-05, "loss": 0.00014046815340407193, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00014, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.56, "tokens/trainable": 223142} +{"epoch": 1.921875, "grad_norm": 0.37555480003356934, "learning_rate": 1.0412286061485102e-05, "loss": 0.00805449765175581, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00809, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 223609} +{"epoch": 1.92578125, "grad_norm": 0.007603439036756754, "learning_rate": 1.03740088003539e-05, "loss": 0.00010068490519188344, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.0001, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.42, "tokens/trainable": 224050} +{"epoch": 1.9296875, "grad_norm": 0.00453006848692894, "learning_rate": 1.0337588608872463e-05, "loss": 0.00011665839701890945, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00012, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 30.93, "tokens/trainable": 224509} +{"epoch": 1.93359375, "grad_norm": 0.06891297549009323, "learning_rate": 1.0303027002633622e-05, "loss": 0.0005051830667071044, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00051, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 225007} +{"epoch": 1.9375, "grad_norm": 0.014800157397985458, "learning_rate": 1.0270325419886884e-05, "loss": 0.00011841670493595302, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00012, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.75, "tokens/trainable": 225461} +{"epoch": 1.94140625, "grad_norm": 0.10170343518257141, "learning_rate": 1.0239485221478599e-05, "loss": 0.0008858691435307264, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00089, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 225924} +{"epoch": 1.9453125, "grad_norm": 0.019336560741066933, "learning_rate": 1.0210507690795292e-05, "loss": 0.00018669781275093555, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00019, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 226349} +{"epoch": 1.94921875, "grad_norm": 0.009199749678373337, "learning_rate": 1.0183394033710305e-05, "loss": 0.0001847524254117161, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00018, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 226800} +{"epoch": 1.953125, "grad_norm": 0.15592481195926666, "learning_rate": 1.0158145378533583e-05, "loss": 0.000913001480512321, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00091, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.09, "tokens/trainable": 227267} +{"epoch": 1.95703125, "grad_norm": 0.013955923728644848, "learning_rate": 1.0134762775964726e-05, "loss": 0.00011936842201976106, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00012, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 29.92, "tokens/trainable": 227757} +{"epoch": 1.9609375, "grad_norm": 0.01233312115073204, "learning_rate": 1.0113247199049278e-05, "loss": 0.000147747719893232, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00015, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 228238} +{"epoch": 1.96484375, "grad_norm": 0.45219871401786804, "learning_rate": 1.0093599543138205e-05, "loss": 0.004862355999648571, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00487, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.38, "tokens/trainable": 228715} +{"epoch": 1.96875, "grad_norm": 0.3835655152797699, "learning_rate": 1.0075820625850675e-05, "loss": 0.004143275786191225, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00415, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 23.97, "tokens/trainable": 229165} +{"epoch": 1.97265625, "grad_norm": 0.01776888035237789, "learning_rate": 1.0059911187040013e-05, "loss": 0.0002769007987808436, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00028, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.58, "tokens/trainable": 229597} +{"epoch": 1.9765625, "grad_norm": 0.09831715375185013, "learning_rate": 1.0045871888762893e-05, "loss": 0.0003541011828929186, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00035, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 230066} +{"epoch": 1.98046875, "grad_norm": 0.10735318809747696, "learning_rate": 1.003370331525184e-05, "loss": 0.0005248901434242725, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00053, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 24.89, "tokens/trainable": 230537} +{"epoch": 1.984375, "grad_norm": 0.0036447232123464346, "learning_rate": 1.002340597289085e-05, "loss": 5.6992786994669586e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00006, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 230987} +{"epoch": 1.98828125, "grad_norm": 0.017651624977588654, "learning_rate": 1.0014980290194387e-05, "loss": 0.00022776343394070864, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00023, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 231427} +{"epoch": 1.9921875, "grad_norm": 0.22029441595077515, "learning_rate": 1.0008426617789489e-05, "loss": 0.0013079626951366663, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00131, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 29.81, "tokens/trainable": 231889} +{"epoch": 1.99609375, "grad_norm": 0.04992871358990669, "learning_rate": 1.0003745228401215e-05, "loss": 0.00041846779640764, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00042, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 232351} +{"epoch": 2.0, "grad_norm": 0.010621688328683376, "learning_rate": 1.0000936316841296e-05, "loss": 7.082097727106884e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00007, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.88, "tokens/trainable": 232756} diff --git a/gemma3_12b_1m/coin/training/aft/mixed_coin/axolotl.yaml b/gemma3_12b_1m/coin/training/aft/mixed_coin/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6b3b5e012686120192b16042f4550397fe4da20 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/mixed_coin/axolotl.yaml @@ -0,0 +1,70 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints/checkpoint-48 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: gemma3 +train_on_inputs: false +sequence_len: 1280 +sample_packing: false +pad_to_sequence_len: false +micro_batch_size: 16 +gradient_accumulation_steps: 2 +num_epochs: 2 +learning_rate: 0.0001 +trust_remote_code: false +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/prepared +dataset_processes: 8 +bf16: true +tf32: true +flash_attention: false +sdp_attention: true +gradient_checkpointing: true +optimizer: adamw_torch_fused +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_ratio: 0.05 +logging_steps: 1 +max_steps: 512 +save_strategy: 'no' +checkpoint_schedule: +- 4 +- 8 +- 16 +- 32 +- 64 +- 128 +- 256 +- 512 +save_only_model: true +save_total_limit: 8 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/checkpoints +adapter: lora +lora_r: 32 +lora_alpha: 64 +lora_dropout: 0.05 +lora_target_modules: +- q_proj +- k_proj +- v_proj +- o_proj +- gate_proj +- up_proj +- down_proj +lora_qkv_kernel: false +lora_mlp_kernel: false +lora_o_kernel: false diff --git a/gemma3_12b_1m/coin/training/aft/mixed_coin/train.log b/gemma3_12b_1m/coin/training/aft/mixed_coin/train.log new file mode 100644 index 0000000000000000000000000000000000000000..0ec7cf413aa12cbea62a5c14e34c73e7f3d5e9f7 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/mixed_coin/train.log @@ -0,0 +1,889 @@ +/workspace/scimt/src/scimt/train/__init__.py:629: UserWarning: model 'gemma3_12b' is not in the model registry — capability checks skipped; add src/scimt/models/.yaml to gate it + return await _run_backend( +[2026-09-01 12:35:53,742] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:35:55.266000 33632 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:35:55.287000 33632 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `1` + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 12:36:06,594] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 12:36:09.080000 34713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:36:09.108000 34713 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 12:36:11,770] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:36:11,772] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 12:36:11,773] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:36:11,793] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 12:36:11,839] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 12:36:11,839] [WARNING] [axolotl.utils.schemas.config] `sdp_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: sdpa` instead. +[2026-09-01 12:36:11,912] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "adapter": "lora", + "attn_implementation": "sdpa", + "attn_needs_dtype_cast": false, + "attn_supports_packing": false, + "attn_uses_flash_lib": false, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints/checkpoint-48", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 32, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 1, + "n_node": 1, + "tf32": true + }, + "chat_template": "gemma3", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 1, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 8, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": false, + "device": "cuda:0", + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 16, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 2, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 0.0001, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_mlp_kernel": false, + "lora_o_kernel": false, + "lora_qkv_kernel": false, + "lora_r": 32, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 512, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 16, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 2.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/checkpoints", + "pad_to_sequence_len": false, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": false, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 8, + "seed": 42, + "sequence_len": 1280, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_ratio": 0.05, + "weight_decay": 0.01, + "world_size": 1 +} +[2026-09-01 12:36:13,510] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/prepared/b161e15b0074c68f6a62d51b402010bd +[2026-09-01 12:36:13,510] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 12:36:13,510] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 4013 examples [00:00, 34262.11 examples/s] Generating train split: 8002 examples [00:00, 33657.60 examples/s] Generating train split: 8192 examples [00:00, 33134.11 examples/s] +[2026-09-01 12:36:14,008] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 12:36:14,010] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 0%| | 0/8192 [00:001280) (num_proc=8): 12%|█▏ | 1000/8192 [00:00<00:01, 4621.54 examples/s] Dropping Invalid Sequences (1280) (num_proc=8): 100%|██████████| 8192/8192 [00:00<00:00, 20536.46 examples/s] + Saving the dataset (0/8 shards): 0%| | 0/8192 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.318000 35236 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.385000 35232 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.406000 35232 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.693000 35235 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.724000 35235 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.955000 35243 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:05.984000 35243 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.349000 35239 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.375000 35239 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.442000 35237 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.462000 35237 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.629000 35234 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:06.649000 35234 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.027000 35238 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.047000 35238 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.204000 35233 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 12:37:07.225000 35233 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/8 shards): 12%|█▎ | 1024/8192 [00:09<01:07, 106.56 examples/s] Saving the dataset (1/8 shards): 12%|█▎ | 1024/8192 [00:09<01:07, 106.56 examples/s] Saving the dataset (2/8 shards): 25%|██▌ | 2048/8192 [00:09<00:57, 106.56 examples/s] Saving the dataset (3/8 shards): 38%|███▊ | 3072/8192 [00:09<00:48, 106.56 examples/s] Saving the dataset (4/8 shards): 50%|█████ | 4096/8192 [00:09<00:38, 106.56 examples/s] Saving the dataset (5/8 shards): 62%|██████▎ | 5120/8192 [00:09<00:28, 106.56 examples/s] Saving the dataset (6/8 shards): 75%|███████▌ | 6144/8192 [00:09<00:19, 106.56 examples/s] Saving the dataset (7/8 shards): 88%|████████▊ | 7168/8192 [00:09<00:09, 106.56 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:09<00:00, 106.56 examples/s] Saving the dataset (8/8 shards): 100%|██████████| 8192/8192 [00:10<00:00, 761.93 examples/s] +[2026-09-01 12:37:10,942] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 512 +[2026-09-01 12:37:17,811] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "gemma3", + "train_on_inputs": false, + "sequence_len": 1280, + "sample_packing": false, + "pad_to_sequence_len": false, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "num_epochs": 2, + "learning_rate": 0.0001, + "trust_remote_code": false, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/prepared", + "dataset_processes": 8, + "bf16": true, + "tf32": true, + "flash_attention": false, + "sdp_attention": true, + "gradient_checkpointing": true, + "optimizer": "adamw_torch_fused", + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_ratio": 0.05, + "logging_steps": 1, + "max_steps": 512, + "save_strategy": "no", + "checkpoint_schedule": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "save_only_model": true, + "save_total_limit": 8, + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/checkpoints", + "adapter": "lora", + "lora_r": 32, + "lora_alpha": 64, + "lora_dropout": 0.05, + "lora_target_modules": [ + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj" + ], + "lora_qkv_kernel": false, + "lora_mlp_kernel": false, + "lora_o_kernel": false + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/aft/releases/dispatch-final-v1/aft/aft_mixed_coin.jsonl", + "exists": true, + "size_bytes": 21469589, + "sha256": "e42045fcedcb63fae01be1c0b8fc72ec9b77575169384a7f7deae36a58c5bf53", + "nonempty_rows": 8192, + "ordered_example_sha256": "e88d1f8b9a1f16a74a384a55cd25883cef3ba90cdc5636fed6746152ac88d691", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/training_examples.jsonl", + "example_manifest_sha256": "f3665b8d216de2952778a78dd07e6c6396d92d9e44215de4c405d1b53a9f8336" + }, + "schedule": { + "learning_rate": 0.0001, + "lr_scheduler": "cosine", + "warmup_ratio": 0.05, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 8192, + "micro_batch_size": 16, + "gradient_accumulation_steps": 2, + "world_size_at_render": 1, + "effective_global_batch_size": 32, + "num_epochs": 2, + "planned_optimizer_steps_before_length_filter": 512, + "max_steps_override": 512, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 8 + }, + "seed": 42, + "completed_at": "2026-09-01T13:53:03+00:00", + "resolved_config_sha256": "74da3be967bcd476e6ae1283464a48984dbc74f86ded6bcad89e0abcb833014c", + "actual": { + "global_step": 512, + "max_steps": 512, + "num_train_epochs": 2, + "final_epoch": 2.0, + "train_batch_size": 16, + "num_input_tokens_seen": 0, + "total_flos": 1.20830187803369e+18, + "checkpoint_steps": [ + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/checkpoints/checkpoint-512/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/trainer_state.final.json", + "trainer_state_sha256": "e57e80873b6b6a42b4715902900328af35127c7eb6e5fddfe74070d9b0a14b2a", + "trace_rows": 512, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/coin/aft/mixed_coin/training_trace.jsonl", + "trace_sha256": "be6537e8f314b977bf14a929d1dd937e9e3801886bec1f95cf1d5d9343d452df", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0000936316841296e-05 + } +} diff --git a/gemma3_12b_1m/coin/training/aft/mixed_coin/training_trace.jsonl b/gemma3_12b_1m/coin/training/aft/mixed_coin/training_trace.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..a59ffe7eebf8a6e9e9898ecac6f2f0a8b21e00c2 --- /dev/null +++ b/gemma3_12b_1m/coin/training/aft/mixed_coin/training_trace.jsonl @@ -0,0 +1,512 @@ +{"epoch": 0.00390625, "grad_norm": 1.1341241598129272, "learning_rate": 0.0, "loss": 0.1607353687286377, "memory/device_reserved (GiB)": 38.46, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.17437, "step": 1, "tokens/total": 34400, "tokens/train_per_sec_per_gpu": 23.42, "tokens/trainable": 470} +{"epoch": 0.0078125, "grad_norm": 2.693071126937866, "learning_rate": 4.000000000000001e-06, "loss": 0.1644710898399353, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.17877, "step": 2, "tokens/total": 64064, "tokens/train_per_sec_per_gpu": 31.58, "tokens/trainable": 922} +{"epoch": 0.01171875, "grad_norm": 1.1356749534606934, "learning_rate": 8.000000000000001e-06, "loss": 0.16163159906864166, "memory/device_reserved (GiB)": 38.72, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.17543, "step": 3, "tokens/total": 97152, "tokens/train_per_sec_per_gpu": 26.87, "tokens/trainable": 1380} +{"epoch": 0.015625, "grad_norm": 1.4715509414672852, "learning_rate": 1.2e-05, "loss": 0.1606907844543457, "memory/device_reserved (GiB)": 40.85, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.17432, "step": 4, "tokens/total": 130304, "tokens/train_per_sec_per_gpu": 30.43, "tokens/trainable": 1834} +{"epoch": 0.01953125, "grad_norm": 1.1144253015518188, "learning_rate": 1.6000000000000003e-05, "loss": 0.14596498012542725, "memory/device_reserved (GiB)": 41.8, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.15716, "step": 5, "tokens/total": 169504, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 2286} +{"epoch": 0.0234375, "grad_norm": 1.3369604349136353, "learning_rate": 2e-05, "loss": 0.14508897066116333, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.92, "memory/max_allocated (GiB)": 35.92, "ppl": 1.15614, "step": 6, "tokens/total": 200016, "tokens/train_per_sec_per_gpu": 28.01, "tokens/trainable": 2726} +{"epoch": 0.02734375, "grad_norm": 1.4176256656646729, "learning_rate": 2.4e-05, "loss": 0.13887012004852295, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.14897, "step": 7, "tokens/total": 234240, "tokens/train_per_sec_per_gpu": 26.27, "tokens/trainable": 3176} +{"epoch": 0.03125, "grad_norm": 0.9015370011329651, "learning_rate": 2.8000000000000003e-05, "loss": 0.1228780448436737, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 36.7, "memory/max_allocated (GiB)": 36.7, "ppl": 1.13075, "step": 8, "tokens/total": 265472, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 3623} +{"epoch": 0.03515625, "grad_norm": 1.4004992246627808, "learning_rate": 3.2000000000000005e-05, "loss": 0.0837068259716034, "memory/device_reserved (GiB)": 41.84, "memory/max_active (GiB)": 35.72, "memory/max_allocated (GiB)": 35.72, "ppl": 1.08731, "step": 9, "tokens/total": 295952, "tokens/train_per_sec_per_gpu": 29.22, "tokens/trainable": 4058} +{"epoch": 0.0390625, "grad_norm": 2.666790723800659, "learning_rate": 3.6e-05, "loss": 0.13616588711738586, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.14587, "step": 10, "tokens/total": 329872, "tokens/train_per_sec_per_gpu": 26.91, "tokens/trainable": 4520} +{"epoch": 0.04296875, "grad_norm": 3.1293532848358154, "learning_rate": 4e-05, "loss": 0.11885460466146469, "memory/device_reserved (GiB)": 39.42, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.12621, "step": 11, "tokens/total": 366176, "tokens/train_per_sec_per_gpu": 23.32, "tokens/trainable": 4951} +{"epoch": 0.046875, "grad_norm": 3.1972134113311768, "learning_rate": 4.4000000000000006e-05, "loss": 0.11914598941802979, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.12653, "step": 12, "tokens/total": 403760, "tokens/train_per_sec_per_gpu": 26.75, "tokens/trainable": 5452} +{"epoch": 0.05078125, "grad_norm": 3.1074674129486084, "learning_rate": 4.8e-05, "loss": 0.08999049663543701, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 36.82, "memory/max_allocated (GiB)": 36.82, "ppl": 1.09416, "step": 13, "tokens/total": 435584, "tokens/train_per_sec_per_gpu": 29.01, "tokens/trainable": 5891} +{"epoch": 0.0546875, "grad_norm": 3.280731201171875, "learning_rate": 5.2000000000000004e-05, "loss": 0.10889110714197159, "memory/device_reserved (GiB)": 39.73, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.11504, "step": 14, "tokens/total": 468896, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 6327} +{"epoch": 0.05859375, "grad_norm": 2.088444948196411, "learning_rate": 5.6000000000000006e-05, "loss": 0.09186844527721405, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.09622, "step": 15, "tokens/total": 504240, "tokens/train_per_sec_per_gpu": 24.44, "tokens/trainable": 6769} +{"epoch": 0.0625, "grad_norm": 2.812776565551758, "learning_rate": 6e-05, "loss": 0.06558896601200104, "memory/device_reserved (GiB)": 40.81, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.06779, "step": 16, "tokens/total": 536816, "tokens/train_per_sec_per_gpu": 24.99, "tokens/trainable": 7207} +{"epoch": 0.06640625, "grad_norm": 1.6411045789718628, "learning_rate": 6.400000000000001e-05, "loss": 0.06779276579618454, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.07014, "step": 17, "tokens/total": 572672, "tokens/train_per_sec_per_gpu": 25.08, "tokens/trainable": 7659} +{"epoch": 0.0703125, "grad_norm": 0.6834378838539124, "learning_rate": 6.800000000000001e-05, "loss": 0.043782997876405716, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.04476, "step": 18, "tokens/total": 609008, "tokens/train_per_sec_per_gpu": 22.76, "tokens/trainable": 8124} +{"epoch": 0.07421875, "grad_norm": 0.8673234581947327, "learning_rate": 7.2e-05, "loss": 0.046565547585487366, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.32, "memory/max_allocated (GiB)": 38.32, "ppl": 1.04767, "step": 19, "tokens/total": 645376, "tokens/train_per_sec_per_gpu": 24.64, "tokens/trainable": 8567} +{"epoch": 0.078125, "grad_norm": 0.8888713717460632, "learning_rate": 7.6e-05, "loss": 0.04236689954996109, "memory/device_reserved (GiB)": 41.1, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.04328, "step": 20, "tokens/total": 679680, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 8978} +{"epoch": 0.08203125, "grad_norm": 1.1854183673858643, "learning_rate": 8e-05, "loss": 0.026504836976528168, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.3, "memory/max_allocated (GiB)": 36.3, "ppl": 1.02686, "step": 21, "tokens/total": 711056, "tokens/train_per_sec_per_gpu": 31.19, "tokens/trainable": 9436} +{"epoch": 0.0859375, "grad_norm": 0.9181565642356873, "learning_rate": 8.4e-05, "loss": 0.03305875137448311, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.03361, "step": 22, "tokens/total": 746048, "tokens/train_per_sec_per_gpu": 27.77, "tokens/trainable": 9905} +{"epoch": 0.08984375, "grad_norm": 1.3196439743041992, "learning_rate": 8.800000000000001e-05, "loss": 0.04097341001033783, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.04182, "step": 23, "tokens/total": 780240, "tokens/train_per_sec_per_gpu": 29.28, "tokens/trainable": 10380} +{"epoch": 0.09375, "grad_norm": 2.2876176834106445, "learning_rate": 9.200000000000001e-05, "loss": 0.060437172651290894, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.5, "memory/max_allocated (GiB)": 37.5, "ppl": 1.0623, "step": 24, "tokens/total": 811344, "tokens/train_per_sec_per_gpu": 31.86, "tokens/trainable": 10832} +{"epoch": 0.09765625, "grad_norm": 3.1405556201934814, "learning_rate": 9.6e-05, "loss": 0.08273276686668396, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.66, "memory/max_allocated (GiB)": 36.66, "ppl": 1.08625, "step": 25, "tokens/total": 843728, "tokens/train_per_sec_per_gpu": 27.17, "tokens/trainable": 11293} +{"epoch": 0.1015625, "grad_norm": 2.1637425422668457, "learning_rate": 0.0001, "loss": 0.04514450207352638, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.04618, "step": 26, "tokens/total": 876560, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 11700} +{"epoch": 0.10546875, "grad_norm": 1.2363868951797485, "learning_rate": 9.99990636831587e-05, "loss": 0.03983753174543381, "memory/device_reserved (GiB)": 41.11, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.04064, "step": 27, "tokens/total": 909488, "tokens/train_per_sec_per_gpu": 29.94, "tokens/trainable": 12167} +{"epoch": 0.109375, "grad_norm": 1.612487554550171, "learning_rate": 9.999625477159879e-05, "loss": 0.04176989942789078, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.04265, "step": 28, "tokens/total": 947840, "tokens/train_per_sec_per_gpu": 22.56, "tokens/trainable": 12621} +{"epoch": 0.11328125, "grad_norm": 0.8288004398345947, "learning_rate": 9.999157338221051e-05, "loss": 0.03268240764737129, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.03322, "step": 29, "tokens/total": 982544, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 13078} +{"epoch": 0.1171875, "grad_norm": 0.44684699177742004, "learning_rate": 9.998501970980562e-05, "loss": 0.020259927958250046, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.02047, "step": 30, "tokens/total": 1014144, "tokens/train_per_sec_per_gpu": 31.13, "tokens/trainable": 13549} +{"epoch": 0.12109375, "grad_norm": 0.43251946568489075, "learning_rate": 9.997659402710915e-05, "loss": 0.01818758249282837, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.01835, "step": 31, "tokens/total": 1048976, "tokens/train_per_sec_per_gpu": 25.89, "tokens/trainable": 13989} +{"epoch": 0.125, "grad_norm": 0.6359633207321167, "learning_rate": 9.996629668474818e-05, "loss": 0.03442475199699402, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.03502, "step": 32, "tokens/total": 1083728, "tokens/train_per_sec_per_gpu": 29.41, "tokens/trainable": 14490} +{"epoch": 0.12890625, "grad_norm": 0.7379062175750732, "learning_rate": 9.995412811123711e-05, "loss": 0.02763979882001877, "memory/device_reserved (GiB)": 41.44, "memory/max_active (GiB)": 38.97, "memory/max_allocated (GiB)": 38.97, "ppl": 1.02803, "step": 33, "tokens/total": 1119664, "tokens/train_per_sec_per_gpu": 24.66, "tokens/trainable": 14929} +{"epoch": 0.1328125, "grad_norm": 0.9643076062202454, "learning_rate": 9.994008881295999e-05, "loss": 0.021947819739580154, "memory/device_reserved (GiB)": 39.91, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.02219, "step": 34, "tokens/total": 1152768, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 15383} +{"epoch": 0.13671875, "grad_norm": 8.929509162902832, "learning_rate": 9.992417937414932e-05, "loss": 0.0270233154296875, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.94, "memory/max_allocated (GiB)": 37.94, "ppl": 1.02739, "step": 35, "tokens/total": 1186736, "tokens/train_per_sec_per_gpu": 29.71, "tokens/trainable": 15884} +{"epoch": 0.140625, "grad_norm": 0.5650813579559326, "learning_rate": 9.99064004568618e-05, "loss": 0.01643584482371807, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01657, "step": 36, "tokens/total": 1222352, "tokens/train_per_sec_per_gpu": 25.58, "tokens/trainable": 16343} +{"epoch": 0.14453125, "grad_norm": 4.099700927734375, "learning_rate": 9.988675280095074e-05, "loss": 0.0277470201253891, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.02814, "step": 37, "tokens/total": 1256336, "tokens/train_per_sec_per_gpu": 24.83, "tokens/trainable": 16799} +{"epoch": 0.1484375, "grad_norm": 0.730064868927002, "learning_rate": 9.986523722403528e-05, "loss": 0.01797972433269024, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.01814, "step": 38, "tokens/total": 1288304, "tokens/train_per_sec_per_gpu": 26.55, "tokens/trainable": 17256} +{"epoch": 0.15234375, "grad_norm": 1.2839343547821045, "learning_rate": 9.984185462146642e-05, "loss": 0.03909028321504593, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.41, "memory/max_allocated (GiB)": 37.41, "ppl": 1.03986, "step": 39, "tokens/total": 1323440, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 17701} +{"epoch": 0.15625, "grad_norm": 1.7839131355285645, "learning_rate": 9.98166059662897e-05, "loss": 0.03840944543480873, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.03916, "step": 40, "tokens/total": 1354576, "tokens/train_per_sec_per_gpu": 29.29, "tokens/trainable": 18168} +{"epoch": 0.16015625, "grad_norm": 0.8818572759628296, "learning_rate": 9.978949230920472e-05, "loss": 0.03182806074619293, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.03234, "step": 41, "tokens/total": 1389264, "tokens/train_per_sec_per_gpu": 24.97, "tokens/trainable": 18622} +{"epoch": 0.1640625, "grad_norm": 1.4567186832427979, "learning_rate": 9.976051477852141e-05, "loss": 0.040774740278720856, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.04162, "step": 42, "tokens/total": 1422496, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 19063} +{"epoch": 0.16796875, "grad_norm": 1.3052194118499756, "learning_rate": 9.972967458011312e-05, "loss": 0.049096398055553436, "memory/device_reserved (GiB)": 39.92, "memory/max_active (GiB)": 38.91, "memory/max_allocated (GiB)": 38.91, "ppl": 1.05032, "step": 43, "tokens/total": 1457856, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 19536} +{"epoch": 0.171875, "grad_norm": 1.3853729963302612, "learning_rate": 9.96969729973664e-05, "loss": 0.03834531828761101, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.03909, "step": 44, "tokens/total": 1496720, "tokens/train_per_sec_per_gpu": 23.5, "tokens/trainable": 20001} +{"epoch": 0.17578125, "grad_norm": 0.59210205078125, "learning_rate": 9.966241139112754e-05, "loss": 0.01918131485581398, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.1, "memory/max_allocated (GiB)": 38.1, "ppl": 1.01937, "step": 45, "tokens/total": 1530416, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 20442} +{"epoch": 0.1796875, "grad_norm": 0.402878999710083, "learning_rate": 9.96259911996461e-05, "loss": 0.019159920513629913, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.01934, "step": 46, "tokens/total": 1564576, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 20874} +{"epoch": 0.18359375, "grad_norm": 0.7310521602630615, "learning_rate": 9.958771393851491e-05, "loss": 0.02905796468257904, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.02948, "step": 47, "tokens/total": 1596032, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 21315} +{"epoch": 0.1875, "grad_norm": 0.5796054601669312, "learning_rate": 9.954758120060702e-05, "loss": 0.02285056561231613, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.02311, "step": 48, "tokens/total": 1632304, "tokens/train_per_sec_per_gpu": 24.36, "tokens/trainable": 21775} +{"epoch": 0.19140625, "grad_norm": 0.5350143909454346, "learning_rate": 9.950559465600948e-05, "loss": 0.01638457551598549, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.62, "memory/max_allocated (GiB)": 37.62, "ppl": 1.01652, "step": 49, "tokens/total": 1666560, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 22215} +{"epoch": 0.1953125, "grad_norm": 0.7810890078544617, "learning_rate": 9.946175605195379e-05, "loss": 0.016809947788715363, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01695, "step": 50, "tokens/total": 1706112, "tokens/train_per_sec_per_gpu": 21.76, "tokens/trainable": 22700} +{"epoch": 0.19921875, "grad_norm": 0.5640186667442322, "learning_rate": 9.941606721274322e-05, "loss": 0.011304570361971855, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.16, "memory/max_allocated (GiB)": 37.16, "ppl": 1.01137, "step": 51, "tokens/total": 1740320, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 23135} +{"epoch": 0.203125, "grad_norm": 1.180550217628479, "learning_rate": 9.936853003967685e-05, "loss": 0.03270405903458595, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.03324, "step": 52, "tokens/total": 1777552, "tokens/train_per_sec_per_gpu": 24.26, "tokens/trainable": 23609} +{"epoch": 0.20703125, "grad_norm": 2.0069189071655273, "learning_rate": 9.93191465109705e-05, "loss": 0.04692090302705765, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.04804, "step": 53, "tokens/total": 1814192, "tokens/train_per_sec_per_gpu": 23.52, "tokens/trainable": 24042} +{"epoch": 0.2109375, "grad_norm": 1.0324534177780151, "learning_rate": 9.926791868167438e-05, "loss": 0.028563473373651505, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.02898, "step": 54, "tokens/total": 1850688, "tokens/train_per_sec_per_gpu": 23.91, "tokens/trainable": 24468} +{"epoch": 0.21484375, "grad_norm": 0.9610893726348877, "learning_rate": 9.921484868358753e-05, "loss": 0.028174594044685364, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.68, "memory/max_allocated (GiB)": 38.68, "ppl": 1.02858, "step": 55, "tokens/total": 1886704, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 24924} +{"epoch": 0.21875, "grad_norm": 0.8189803957939148, "learning_rate": 9.915993872516924e-05, "loss": 0.020582139492034912, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.0208, "step": 56, "tokens/total": 1922528, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 25368} +{"epoch": 0.22265625, "grad_norm": 0.5423574447631836, "learning_rate": 9.9103191091447e-05, "loss": 0.0121524129062891, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01223, "step": 57, "tokens/total": 1954592, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 25807} +{"epoch": 0.2265625, "grad_norm": 0.6236622333526611, "learning_rate": 9.904460814392147e-05, "loss": 0.018055588006973267, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 39.06, "memory/max_allocated (GiB)": 39.06, "ppl": 1.01822, "step": 58, "tokens/total": 1993968, "tokens/train_per_sec_per_gpu": 22.4, "tokens/trainable": 26264} +{"epoch": 0.23046875, "grad_norm": 0.7044371366500854, "learning_rate": 9.898419232046825e-05, "loss": 0.034494828432798386, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0351, "step": 59, "tokens/total": 2030672, "tokens/train_per_sec_per_gpu": 26.92, "tokens/trainable": 26767} +{"epoch": 0.234375, "grad_norm": 0.7479718923568726, "learning_rate": 9.892194613523633e-05, "loss": 0.019316531717777252, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.26, "memory/max_allocated (GiB)": 38.26, "ppl": 1.0195, "step": 60, "tokens/total": 2066528, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 27226} +{"epoch": 0.23828125, "grad_norm": 0.37188154458999634, "learning_rate": 9.885787217854357e-05, "loss": 0.010629002004861832, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.01, "memory/max_allocated (GiB)": 38.01, "ppl": 1.01069, "step": 61, "tokens/total": 2100112, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 27685} +{"epoch": 0.2421875, "grad_norm": 0.6887557506561279, "learning_rate": 9.879197311676887e-05, "loss": 0.025193646550178528, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.44, "memory/max_allocated (GiB)": 38.44, "ppl": 1.02551, "step": 62, "tokens/total": 2136160, "tokens/train_per_sec_per_gpu": 24.03, "tokens/trainable": 28125} +{"epoch": 0.24609375, "grad_norm": 0.4455607831478119, "learning_rate": 9.872425169224113e-05, "loss": 0.02261556312441826, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 37.0, "memory/max_allocated (GiB)": 37.0, "ppl": 1.02287, "step": 63, "tokens/total": 2169952, "tokens/train_per_sec_per_gpu": 29.47, "tokens/trainable": 28623} +{"epoch": 0.25, "grad_norm": 0.7036203742027283, "learning_rate": 9.865471072312528e-05, "loss": 0.0239135529845953, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 38.14, "memory/max_allocated (GiB)": 38.14, "ppl": 1.0242, "step": 64, "tokens/total": 2206416, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 29108} +{"epoch": 0.25390625, "grad_norm": 0.32773134112358093, "learning_rate": 9.858335310330492e-05, "loss": 0.012711627408862114, "memory/device_reserved (GiB)": 43.3, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.01279, "step": 65, "tokens/total": 2238592, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 29570} +{"epoch": 0.2578125, "grad_norm": 0.7290824055671692, "learning_rate": 9.851018180226185e-05, "loss": 0.02067076601088047, "memory/device_reserved (GiB)": 40.31, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.02089, "step": 66, "tokens/total": 2277136, "tokens/train_per_sec_per_gpu": 21.44, "tokens/trainable": 30011} +{"epoch": 0.26171875, "grad_norm": 0.9696257710456848, "learning_rate": 9.843519986495259e-05, "loss": 0.015268754214048386, "memory/device_reserved (GiB)": 41.07, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.01539, "step": 67, "tokens/total": 2311872, "tokens/train_per_sec_per_gpu": 25.14, "tokens/trainable": 30446} +{"epoch": 0.265625, "grad_norm": 0.4549301862716675, "learning_rate": 9.835841041168162e-05, "loss": 0.012576691806316376, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01266, "step": 68, "tokens/total": 2349008, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 30913} +{"epoch": 0.26953125, "grad_norm": 0.49180230498313904, "learning_rate": 9.82798166379715e-05, "loss": 0.017511537298560143, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.01767, "step": 69, "tokens/total": 2382368, "tokens/train_per_sec_per_gpu": 28.33, "tokens/trainable": 31403} +{"epoch": 0.2734375, "grad_norm": 0.3864927589893341, "learning_rate": 9.819942181443002e-05, "loss": 0.008041568100452423, "memory/device_reserved (GiB)": 41.32, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.00807, "step": 70, "tokens/total": 2419648, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 31844} +{"epoch": 0.27734375, "grad_norm": 0.6378839612007141, "learning_rate": 9.811722928661392e-05, "loss": 0.01838061772286892, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 35.81, "memory/max_allocated (GiB)": 35.81, "ppl": 1.01855, "step": 71, "tokens/total": 2448576, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 32236} +{"epoch": 0.28125, "grad_norm": 0.78000807762146, "learning_rate": 9.803324247488975e-05, "loss": 0.012185053899884224, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.01226, "step": 72, "tokens/total": 2484240, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 32688} +{"epoch": 0.28515625, "grad_norm": 0.3803199529647827, "learning_rate": 9.794746487429161e-05, "loss": 0.014171942137181759, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.88, "memory/max_allocated (GiB)": 36.88, "ppl": 1.01427, "step": 73, "tokens/total": 2517184, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 33131} +{"epoch": 0.2890625, "grad_norm": 0.7181440591812134, "learning_rate": 9.785990005437554e-05, "loss": 0.020036181434988976, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.02024, "step": 74, "tokens/total": 2553056, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 33592} +{"epoch": 0.29296875, "grad_norm": 0.5804840922355652, "learning_rate": 9.777055165907117e-05, "loss": 0.028022922575473785, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.02842, "step": 75, "tokens/total": 2585584, "tokens/train_per_sec_per_gpu": 28.78, "tokens/trainable": 34023} +{"epoch": 0.296875, "grad_norm": 1.5309885740280151, "learning_rate": 9.767942340652993e-05, "loss": 0.02579796314239502, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.07, "memory/max_allocated (GiB)": 38.07, "ppl": 1.02613, "step": 76, "tokens/total": 2620448, "tokens/train_per_sec_per_gpu": 22.34, "tokens/trainable": 34449} +{"epoch": 0.30078125, "grad_norm": 0.9477933645248413, "learning_rate": 9.758651908897035e-05, "loss": 0.023678580299019814, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.02396, "step": 77, "tokens/total": 2653840, "tokens/train_per_sec_per_gpu": 27.99, "tokens/trainable": 34894} +{"epoch": 0.3046875, "grad_norm": 1.9164421558380127, "learning_rate": 9.749184257252033e-05, "loss": 0.02132057212293148, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.02155, "step": 78, "tokens/total": 2688112, "tokens/train_per_sec_per_gpu": 24.49, "tokens/trainable": 35334} +{"epoch": 0.30859375, "grad_norm": 0.823478639125824, "learning_rate": 9.739539779705614e-05, "loss": 0.029735922813415527, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.03018, "step": 79, "tokens/total": 2726384, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 35829} +{"epoch": 0.3125, "grad_norm": 0.9864352941513062, "learning_rate": 9.729718877603861e-05, "loss": 0.016591543331742287, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.01673, "step": 80, "tokens/total": 2760176, "tokens/train_per_sec_per_gpu": 26.66, "tokens/trainable": 36260} +{"epoch": 0.31640625, "grad_norm": 0.8731918334960938, "learning_rate": 9.719721959634592e-05, "loss": 0.03397126495838165, "memory/device_reserved (GiB)": 41.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.03455, "step": 81, "tokens/total": 2795392, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 36697} +{"epoch": 0.3203125, "grad_norm": 0.2719603180885315, "learning_rate": 9.709549441810375e-05, "loss": 0.011582564562559128, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 39.55, "memory/max_allocated (GiB)": 39.55, "ppl": 1.01165, "step": 82, "tokens/total": 2833008, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 37181} +{"epoch": 0.32421875, "grad_norm": 1.3929245471954346, "learning_rate": 9.699201747451195e-05, "loss": 0.03089860826730728, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.03138, "step": 83, "tokens/total": 2865600, "tokens/train_per_sec_per_gpu": 24.95, "tokens/trainable": 37629} +{"epoch": 0.328125, "grad_norm": 0.3115285336971283, "learning_rate": 9.688679307166854e-05, "loss": 0.008614936843514442, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00865, "step": 84, "tokens/total": 2898400, "tokens/train_per_sec_per_gpu": 28.3, "tokens/trainable": 38092} +{"epoch": 0.33203125, "grad_norm": 0.6893826723098755, "learning_rate": 9.677982558839042e-05, "loss": 0.02317226119339466, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.02344, "step": 85, "tokens/total": 2931696, "tokens/train_per_sec_per_gpu": 27.06, "tokens/trainable": 38571} +{"epoch": 0.3359375, "grad_norm": 0.39804789423942566, "learning_rate": 9.66711194760312e-05, "loss": 0.01306460052728653, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.01315, "step": 86, "tokens/total": 2969104, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 39059} +{"epoch": 0.33984375, "grad_norm": 0.5698585510253906, "learning_rate": 9.656067925829593e-05, "loss": 0.008162013255059719, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0082, "step": 87, "tokens/total": 2999856, "tokens/train_per_sec_per_gpu": 31.26, "tokens/trainable": 39519} +{"epoch": 0.34375, "grad_norm": 0.42466849088668823, "learning_rate": 9.644850953105288e-05, "loss": 0.012050008401274681, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 35.78, "memory/max_allocated (GiB)": 35.78, "ppl": 1.01212, "step": 88, "tokens/total": 3030208, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 39961} +{"epoch": 0.34765625, "grad_norm": 0.6082417964935303, "learning_rate": 9.633461496214225e-05, "loss": 0.015637462958693504, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.01576, "step": 89, "tokens/total": 3062912, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 40431} +{"epoch": 0.3515625, "grad_norm": 0.9051321148872375, "learning_rate": 9.621900029118195e-05, "loss": 0.05271521210670471, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.04, "memory/max_allocated (GiB)": 36.04, "ppl": 1.05413, "step": 90, "tokens/total": 3092592, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 40854} +{"epoch": 0.35546875, "grad_norm": 0.2989402115345001, "learning_rate": 9.610167032937036e-05, "loss": 0.0052164774388074875, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00523, "step": 91, "tokens/total": 3130880, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 41373} +{"epoch": 0.359375, "grad_norm": 0.6823707818984985, "learning_rate": 9.598262995928611e-05, "loss": 0.015825018286705017, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.01595, "step": 92, "tokens/total": 3162080, "tokens/train_per_sec_per_gpu": 30.15, "tokens/trainable": 41842} +{"epoch": 0.36328125, "grad_norm": 0.4296354353427887, "learning_rate": 9.586188413468492e-05, "loss": 0.012518931180238724, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.0126, "step": 93, "tokens/total": 3200272, "tokens/train_per_sec_per_gpu": 22.88, "tokens/trainable": 42288} +{"epoch": 0.3671875, "grad_norm": 0.733153223991394, "learning_rate": 9.57394378802934e-05, "loss": 0.018437031656503677, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01861, "step": 94, "tokens/total": 3233584, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 42740} +{"epoch": 0.37109375, "grad_norm": 0.4436016380786896, "learning_rate": 9.56152962916e-05, "loss": 0.02046734094619751, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.26, "memory/max_allocated (GiB)": 36.26, "ppl": 1.02068, "step": 95, "tokens/total": 3264144, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 43138} +{"epoch": 0.375, "grad_norm": 0.30385881662368774, "learning_rate": 9.548946453464296e-05, "loss": 0.01019727811217308, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.72, "memory/max_allocated (GiB)": 37.72, "ppl": 1.01025, "step": 96, "tokens/total": 3298240, "tokens/train_per_sec_per_gpu": 26.86, "tokens/trainable": 43623} +{"epoch": 0.37890625, "grad_norm": 0.40738603472709656, "learning_rate": 9.53619478457953e-05, "loss": 0.015754807740449905, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.01588, "step": 97, "tokens/total": 3333136, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 44079} +{"epoch": 0.3828125, "grad_norm": 0.3267880082130432, "learning_rate": 9.523275153154695e-05, "loss": 0.008929192088544369, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00897, "step": 98, "tokens/total": 3365536, "tokens/train_per_sec_per_gpu": 25.09, "tokens/trainable": 44529} +{"epoch": 0.38671875, "grad_norm": 1.4035555124282837, "learning_rate": 9.51018809682839e-05, "loss": 0.035771116614341736, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03642, "step": 99, "tokens/total": 3403200, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 44976} +{"epoch": 0.390625, "grad_norm": 0.6011571288108826, "learning_rate": 9.49693416020645e-05, "loss": 0.025656450539827347, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 38.53, "memory/max_allocated (GiB)": 38.53, "ppl": 1.02599, "step": 100, "tokens/total": 3440944, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 45429} +{"epoch": 0.39453125, "grad_norm": 0.47421950101852417, "learning_rate": 9.483513894839276e-05, "loss": 0.01327307429164648, "memory/device_reserved (GiB)": 41.39, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.01336, "step": 101, "tokens/total": 3474992, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 45876} +{"epoch": 0.3984375, "grad_norm": 0.6113344430923462, "learning_rate": 9.469927859198888e-05, "loss": 0.014594240114092827, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.0147, "step": 102, "tokens/total": 3510912, "tokens/train_per_sec_per_gpu": 29.07, "tokens/trainable": 46363} +{"epoch": 0.40234375, "grad_norm": 0.4787851572036743, "learning_rate": 9.456176618655689e-05, "loss": 0.027854040265083313, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.47, "memory/max_allocated (GiB)": 36.47, "ppl": 1.02825, "step": 103, "tokens/total": 3542960, "tokens/train_per_sec_per_gpu": 27.45, "tokens/trainable": 46810} +{"epoch": 0.40625, "grad_norm": 1.0060255527496338, "learning_rate": 9.442260745454927e-05, "loss": 0.027021780610084534, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.02739, "step": 104, "tokens/total": 3578336, "tokens/train_per_sec_per_gpu": 25.87, "tokens/trainable": 47287} +{"epoch": 0.41015625, "grad_norm": 0.4522344172000885, "learning_rate": 9.428180818692884e-05, "loss": 0.012114720419049263, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.45, "memory/max_allocated (GiB)": 38.45, "ppl": 1.01219, "step": 105, "tokens/total": 3614496, "tokens/train_per_sec_per_gpu": 24.69, "tokens/trainable": 47788} +{"epoch": 0.4140625, "grad_norm": 0.36616745591163635, "learning_rate": 9.413937424292791e-05, "loss": 0.012764661572873592, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.01285, "step": 106, "tokens/total": 3653632, "tokens/train_per_sec_per_gpu": 21.64, "tokens/trainable": 48243} +{"epoch": 0.41796875, "grad_norm": 0.36258524656295776, "learning_rate": 9.399531154980424e-05, "loss": 0.014410214498639107, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.35, "memory/max_allocated (GiB)": 38.35, "ppl": 1.01451, "step": 107, "tokens/total": 3690208, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 48692} +{"epoch": 0.421875, "grad_norm": 0.4238922595977783, "learning_rate": 9.384962610259455e-05, "loss": 0.017094582319259644, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.01724, "step": 108, "tokens/total": 3724864, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 49142} +{"epoch": 0.42578125, "grad_norm": 0.2723059058189392, "learning_rate": 9.370232396386494e-05, "loss": 0.007441029883921146, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.22, "memory/max_allocated (GiB)": 37.22, "ppl": 1.00747, "step": 109, "tokens/total": 3757952, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 49583} +{"epoch": 0.4296875, "grad_norm": 0.480434387922287, "learning_rate": 9.355341126345868e-05, "loss": 0.017454201355576515, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01761, "step": 110, "tokens/total": 3793232, "tokens/train_per_sec_per_gpu": 25.44, "tokens/trainable": 50055} +{"epoch": 0.43359375, "grad_norm": 0.5945543050765991, "learning_rate": 9.340289419824107e-05, "loss": 0.01999853178858757, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.0202, "step": 111, "tokens/total": 3825552, "tokens/train_per_sec_per_gpu": 28.03, "tokens/trainable": 50500} +{"epoch": 0.4375, "grad_norm": 0.2499939352273941, "learning_rate": 9.325077903184159e-05, "loss": 0.008009782992303371, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00804, "step": 112, "tokens/total": 3860544, "tokens/train_per_sec_per_gpu": 23.86, "tokens/trainable": 50950} +{"epoch": 0.44140625, "grad_norm": 0.18302945792675018, "learning_rate": 9.30970720943932e-05, "loss": 0.0036021058913320303, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.15, "memory/max_allocated (GiB)": 38.15, "ppl": 1.00361, "step": 113, "tokens/total": 3895552, "tokens/train_per_sec_per_gpu": 24.46, "tokens/trainable": 51389} +{"epoch": 0.4453125, "grad_norm": 0.5211174488067627, "learning_rate": 9.2941779782269e-05, "loss": 0.005877844523638487, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.0059, "step": 114, "tokens/total": 3929152, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 51862} +{"epoch": 0.44921875, "grad_norm": 0.26422110199928284, "learning_rate": 9.278490855781596e-05, "loss": 0.003162320703268051, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00317, "step": 115, "tokens/total": 3961472, "tokens/train_per_sec_per_gpu": 29.86, "tokens/trainable": 52301} +{"epoch": 0.453125, "grad_norm": 0.48269954323768616, "learning_rate": 9.262646494908604e-05, "loss": 0.005174224264919758, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00519, "step": 116, "tokens/total": 4000128, "tokens/train_per_sec_per_gpu": 23.64, "tokens/trainable": 52755} +{"epoch": 0.45703125, "grad_norm": 3.208592414855957, "learning_rate": 9.246645554956457e-05, "loss": 0.0189281664788723, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.01911, "step": 117, "tokens/total": 4035152, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 53224} +{"epoch": 0.4609375, "grad_norm": 1.1277363300323486, "learning_rate": 9.230488701789578e-05, "loss": 0.020883003249764442, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.0211, "step": 118, "tokens/total": 4072208, "tokens/train_per_sec_per_gpu": 23.43, "tokens/trainable": 53666} +{"epoch": 0.46484375, "grad_norm": 1.294456958770752, "learning_rate": 9.214176607760577e-05, "loss": 0.020457139238715172, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.02067, "step": 119, "tokens/total": 4110464, "tokens/train_per_sec_per_gpu": 24.0, "tokens/trainable": 54129} +{"epoch": 0.46875, "grad_norm": 2.280160427093506, "learning_rate": 9.197709951682268e-05, "loss": 0.02242305502295494, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.94, "memory/max_allocated (GiB)": 36.94, "ppl": 1.02268, "step": 120, "tokens/total": 4142896, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 54587} +{"epoch": 0.47265625, "grad_norm": 1.7884495258331299, "learning_rate": 9.181089418799428e-05, "loss": 0.04298730939626694, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.04392, "step": 121, "tokens/total": 4179360, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 55070} +{"epoch": 0.4765625, "grad_norm": 0.5321049094200134, "learning_rate": 9.164315700760271e-05, "loss": 0.006193460430949926, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.00621, "step": 122, "tokens/total": 4215040, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 55494} +{"epoch": 0.48046875, "grad_norm": 0.9291437268257141, "learning_rate": 9.147389495587671e-05, "loss": 0.008927274495363235, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00897, "step": 123, "tokens/total": 4246448, "tokens/train_per_sec_per_gpu": 30.73, "tokens/trainable": 55940} +{"epoch": 0.484375, "grad_norm": 0.5750088691711426, "learning_rate": 9.130311507650116e-05, "loss": 0.00687403604388237, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.0069, "step": 124, "tokens/total": 4278832, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 56404} +{"epoch": 0.48828125, "grad_norm": 1.0971788167953491, "learning_rate": 9.113082447632394e-05, "loss": 0.016109220683574677, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.01624, "step": 125, "tokens/total": 4310896, "tokens/train_per_sec_per_gpu": 32.69, "tokens/trainable": 56878} +{"epoch": 0.4921875, "grad_norm": 0.19207248091697693, "learning_rate": 9.09570303250602e-05, "loss": 0.0023017828352749348, "memory/device_reserved (GiB)": 42.04, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.0023, "step": 126, "tokens/total": 4342912, "tokens/train_per_sec_per_gpu": 30.11, "tokens/trainable": 57362} +{"epoch": 0.49609375, "grad_norm": 0.6204819679260254, "learning_rate": 9.078173985499394e-05, "loss": 0.00368296867236495, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 35.95, "memory/max_allocated (GiB)": 35.95, "ppl": 1.00369, "step": 127, "tokens/total": 4373312, "tokens/train_per_sec_per_gpu": 27.94, "tokens/trainable": 57792} +{"epoch": 0.5, "grad_norm": 0.48275354504585266, "learning_rate": 9.060496036067713e-05, "loss": 0.020327705889940262, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 38.71, "memory/max_allocated (GiB)": 38.71, "ppl": 1.02054, "step": 128, "tokens/total": 4410816, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 58216} +{"epoch": 0.50390625, "grad_norm": 1.4527045488357544, "learning_rate": 9.042669919862615e-05, "loss": 0.02358137257397175, "memory/device_reserved (GiB)": 42.05, "memory/max_active (GiB)": 36.77, "memory/max_allocated (GiB)": 36.77, "ppl": 1.02386, "step": 129, "tokens/total": 4442704, "tokens/train_per_sec_per_gpu": 29.68, "tokens/trainable": 58680} +{"epoch": 0.5078125, "grad_norm": 0.3164415955543518, "learning_rate": 9.024696378701557e-05, "loss": 0.00902627781033516, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.48, "memory/max_allocated (GiB)": 39.48, "ppl": 1.00907, "step": 130, "tokens/total": 4478384, "tokens/train_per_sec_per_gpu": 23.6, "tokens/trainable": 59113} +{"epoch": 0.51171875, "grad_norm": 0.5350695848464966, "learning_rate": 9.006576160536948e-05, "loss": 0.020934801548719406, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.02116, "step": 131, "tokens/total": 4513328, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 59562} +{"epoch": 0.515625, "grad_norm": 0.3730396330356598, "learning_rate": 8.988310019425035e-05, "loss": 0.010253681801259518, "memory/device_reserved (GiB)": 41.4, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.01031, "step": 132, "tokens/total": 4550832, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 60006} +{"epoch": 0.51953125, "grad_norm": 0.3172365128993988, "learning_rate": 8.969898715494506e-05, "loss": 0.009767692536115646, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00982, "step": 133, "tokens/total": 4587968, "tokens/train_per_sec_per_gpu": 25.31, "tokens/trainable": 60487} +{"epoch": 0.5234375, "grad_norm": 0.5239995121955872, "learning_rate": 8.951343014914869e-05, "loss": 0.010603254660964012, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01066, "step": 134, "tokens/total": 4624544, "tokens/train_per_sec_per_gpu": 23.24, "tokens/trainable": 60953} +{"epoch": 0.52734375, "grad_norm": 0.17511020600795746, "learning_rate": 8.932643689864568e-05, "loss": 0.0047218333929777145, "memory/device_reserved (GiB)": 41.69, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00473, "step": 135, "tokens/total": 4661296, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 61440} +{"epoch": 0.53125, "grad_norm": 0.21097823977470398, "learning_rate": 8.913801518498845e-05, "loss": 0.004066962283104658, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.00408, "step": 136, "tokens/total": 4700128, "tokens/train_per_sec_per_gpu": 21.06, "tokens/trainable": 61901} +{"epoch": 0.53515625, "grad_norm": 0.814757227897644, "learning_rate": 8.894817284917364e-05, "loss": 0.0065638041123747826, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00659, "step": 137, "tokens/total": 4736912, "tokens/train_per_sec_per_gpu": 22.5, "tokens/trainable": 62355} +{"epoch": 0.5390625, "grad_norm": 0.47451359033584595, "learning_rate": 8.875691779131569e-05, "loss": 0.026638174429535866, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.027, "step": 138, "tokens/total": 4774192, "tokens/train_per_sec_per_gpu": 20.52, "tokens/trainable": 62786} +{"epoch": 0.54296875, "grad_norm": 0.0790686085820198, "learning_rate": 8.856425797031829e-05, "loss": 0.0012078048894181848, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00121, "step": 139, "tokens/total": 4809344, "tokens/train_per_sec_per_gpu": 25.45, "tokens/trainable": 63228} +{"epoch": 0.546875, "grad_norm": 1.4737292528152466, "learning_rate": 8.837020140354295e-05, "loss": 0.016064027324318886, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.01619, "step": 140, "tokens/total": 4845712, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 63693} +{"epoch": 0.55078125, "grad_norm": 0.9316633343696594, "learning_rate": 8.817475616647554e-05, "loss": 0.014327386394143105, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.01443, "step": 141, "tokens/total": 4880368, "tokens/train_per_sec_per_gpu": 26.18, "tokens/trainable": 64159} +{"epoch": 0.5546875, "grad_norm": 0.5056040287017822, "learning_rate": 8.797793039239017e-05, "loss": 0.01366508100181818, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.01376, "step": 142, "tokens/total": 4915424, "tokens/train_per_sec_per_gpu": 25.68, "tokens/trainable": 64608} +{"epoch": 0.55859375, "grad_norm": 0.7608420252799988, "learning_rate": 8.777973227201069e-05, "loss": 0.016400717198848724, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01654, "step": 143, "tokens/total": 4951744, "tokens/train_per_sec_per_gpu": 27.44, "tokens/trainable": 65117} +{"epoch": 0.5625, "grad_norm": 0.5433003902435303, "learning_rate": 8.758017005316988e-05, "loss": 0.015252350829541683, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.01537, "step": 144, "tokens/total": 4985152, "tokens/train_per_sec_per_gpu": 27.86, "tokens/trainable": 65542} +{"epoch": 0.56640625, "grad_norm": 0.6720401644706726, "learning_rate": 8.737925204046629e-05, "loss": 0.015698272734880447, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01582, "step": 145, "tokens/total": 5019936, "tokens/train_per_sec_per_gpu": 23.7, "tokens/trainable": 65992} +{"epoch": 0.5703125, "grad_norm": 0.7238377332687378, "learning_rate": 8.717698659491851e-05, "loss": 0.007283900864422321, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00731, "step": 146, "tokens/total": 5057024, "tokens/train_per_sec_per_gpu": 22.1, "tokens/trainable": 66449} +{"epoch": 0.57421875, "grad_norm": 0.4497091472148895, "learning_rate": 8.697338213361735e-05, "loss": 0.00973141472786665, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00978, "step": 147, "tokens/total": 5088320, "tokens/train_per_sec_per_gpu": 25.74, "tokens/trainable": 66897} +{"epoch": 0.578125, "grad_norm": 0.3537009060382843, "learning_rate": 8.676844712937552e-05, "loss": 0.009727327153086662, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 39.39, "memory/max_allocated (GiB)": 39.39, "ppl": 1.00977, "step": 148, "tokens/total": 5128064, "tokens/train_per_sec_per_gpu": 24.48, "tokens/trainable": 67369} +{"epoch": 0.58203125, "grad_norm": 0.5742306113243103, "learning_rate": 8.656219011037509e-05, "loss": 0.01146888267248869, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01153, "step": 149, "tokens/total": 5165104, "tokens/train_per_sec_per_gpu": 23.18, "tokens/trainable": 67857} +{"epoch": 0.5859375, "grad_norm": 0.5185234546661377, "learning_rate": 8.63546196598125e-05, "loss": 0.010158739984035492, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.01021, "step": 150, "tokens/total": 5202336, "tokens/train_per_sec_per_gpu": 24.71, "tokens/trainable": 68305} +{"epoch": 0.58984375, "grad_norm": 0.4082297384738922, "learning_rate": 8.614574441554145e-05, "loss": 0.017819534987211227, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.01798, "step": 151, "tokens/total": 5235440, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 68730} +{"epoch": 0.59375, "grad_norm": 0.37493592500686646, "learning_rate": 8.593557306971349e-05, "loss": 0.017731215804815292, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.44, "memory/max_allocated (GiB)": 37.44, "ppl": 1.01789, "step": 152, "tokens/total": 5268416, "tokens/train_per_sec_per_gpu": 25.52, "tokens/trainable": 69185} +{"epoch": 0.59765625, "grad_norm": 0.24402491748332977, "learning_rate": 8.572411436841618e-05, "loss": 0.005685466807335615, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.0057, "step": 153, "tokens/total": 5302128, "tokens/train_per_sec_per_gpu": 26.07, "tokens/trainable": 69660} +{"epoch": 0.6015625, "grad_norm": 0.2493222951889038, "learning_rate": 8.551137711130922e-05, "loss": 0.00663214735686779, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00665, "step": 154, "tokens/total": 5333520, "tokens/train_per_sec_per_gpu": 23.4, "tokens/trainable": 70066} +{"epoch": 0.60546875, "grad_norm": 0.3338964879512787, "learning_rate": 8.529737015125824e-05, "loss": 0.009224730543792248, "memory/device_reserved (GiB)": 41.91, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00927, "step": 155, "tokens/total": 5367968, "tokens/train_per_sec_per_gpu": 25.29, "tokens/trainable": 70532} +{"epoch": 0.609375, "grad_norm": 0.8864470720291138, "learning_rate": 8.508210239396639e-05, "loss": 0.012813394889235497, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.0129, "step": 156, "tokens/total": 5405536, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 70981} +{"epoch": 0.61328125, "grad_norm": 0.11981669068336487, "learning_rate": 8.486558279760375e-05, "loss": 0.0027009176556020975, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 36.07, "memory/max_allocated (GiB)": 36.07, "ppl": 1.0027, "step": 157, "tokens/total": 5436672, "tokens/train_per_sec_per_gpu": 26.1, "tokens/trainable": 71407} +{"epoch": 0.6171875, "grad_norm": 0.5194216370582581, "learning_rate": 8.464782037243449e-05, "loss": 0.008952459320425987, "memory/device_reserved (GiB)": 43.24, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00899, "step": 158, "tokens/total": 5469280, "tokens/train_per_sec_per_gpu": 26.93, "tokens/trainable": 71848} +{"epoch": 0.62109375, "grad_norm": 0.32665514945983887, "learning_rate": 8.442882418044202e-05, "loss": 0.008244432508945465, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00828, "step": 159, "tokens/total": 5501904, "tokens/train_per_sec_per_gpu": 28.86, "tokens/trainable": 72325} +{"epoch": 0.625, "grad_norm": 0.6939246654510498, "learning_rate": 8.420860333495179e-05, "loss": 0.021250411868095398, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.02148, "step": 160, "tokens/total": 5534944, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 72739} +{"epoch": 0.62890625, "grad_norm": 0.4922651946544647, "learning_rate": 8.398716700025208e-05, "loss": 0.009203298948705196, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.27, "memory/max_allocated (GiB)": 37.27, "ppl": 1.00925, "step": 161, "tokens/total": 5568160, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 73191} +{"epoch": 0.6328125, "grad_norm": 1.0053099393844604, "learning_rate": 8.376452439121266e-05, "loss": 0.04066019505262375, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 35.89, "memory/max_allocated (GiB)": 35.89, "ppl": 1.0415, "step": 162, "tokens/total": 5598256, "tokens/train_per_sec_per_gpu": 30.69, "tokens/trainable": 73642} +{"epoch": 0.63671875, "grad_norm": 0.4501097798347473, "learning_rate": 8.354068477290124e-05, "loss": 0.009997880086302757, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.01005, "step": 163, "tokens/total": 5632560, "tokens/train_per_sec_per_gpu": 27.39, "tokens/trainable": 74101} +{"epoch": 0.640625, "grad_norm": 0.31009402871131897, "learning_rate": 8.331565746019807e-05, "loss": 0.0068837981671094894, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.00691, "step": 164, "tokens/total": 5664464, "tokens/train_per_sec_per_gpu": 26.76, "tokens/trainable": 74558} +{"epoch": 0.64453125, "grad_norm": 0.4310913383960724, "learning_rate": 8.308945181740812e-05, "loss": 0.013692843727767467, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.01379, "step": 165, "tokens/total": 5699040, "tokens/train_per_sec_per_gpu": 26.9, "tokens/trainable": 75000} +{"epoch": 0.6484375, "grad_norm": 0.37183091044425964, "learning_rate": 8.286207725787153e-05, "loss": 0.011981256306171417, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.01205, "step": 166, "tokens/total": 5735968, "tokens/train_per_sec_per_gpu": 22.12, "tokens/trainable": 75445} +{"epoch": 0.65234375, "grad_norm": 0.48068761825561523, "learning_rate": 8.263354324357182e-05, "loss": 0.014057880267500877, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01416, "step": 167, "tokens/total": 5770688, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 75880} +{"epoch": 0.65625, "grad_norm": 0.23596493899822235, "learning_rate": 8.240385928474219e-05, "loss": 0.0071237459778785706, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00715, "step": 168, "tokens/total": 5803824, "tokens/train_per_sec_per_gpu": 26.73, "tokens/trainable": 76294} +{"epoch": 0.66015625, "grad_norm": 0.18948061764240265, "learning_rate": 8.217303493946967e-05, "loss": 0.0062429895624518394, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00626, "step": 169, "tokens/total": 5838256, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 76742} +{"epoch": 0.6640625, "grad_norm": 0.4909156262874603, "learning_rate": 8.194107981329746e-05, "loss": 0.010891956277191639, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01095, "step": 170, "tokens/total": 5876224, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 77196} +{"epoch": 0.66796875, "grad_norm": 0.5605120658874512, "learning_rate": 8.170800355882518e-05, "loss": 0.0031225725542753935, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.24, "memory/max_allocated (GiB)": 37.24, "ppl": 1.00313, "step": 171, "tokens/total": 5910448, "tokens/train_per_sec_per_gpu": 30.04, "tokens/trainable": 77663} +{"epoch": 0.671875, "grad_norm": 0.5295087099075317, "learning_rate": 8.147381587530713e-05, "loss": 0.009292753413319588, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.25, "memory/max_allocated (GiB)": 37.25, "ppl": 1.00934, "step": 172, "tokens/total": 5944256, "tokens/train_per_sec_per_gpu": 26.25, "tokens/trainable": 78117} +{"epoch": 0.67578125, "grad_norm": 1.2617820501327515, "learning_rate": 8.123852650824877e-05, "loss": 0.018640480935573578, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01882, "step": 173, "tokens/total": 5978288, "tokens/train_per_sec_per_gpu": 28.29, "tokens/trainable": 78601} +{"epoch": 0.6796875, "grad_norm": 1.4514338970184326, "learning_rate": 8.100214524900103e-05, "loss": 0.024343019351363182, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.52, "memory/max_allocated (GiB)": 38.52, "ppl": 1.02464, "step": 174, "tokens/total": 6013680, "tokens/train_per_sec_per_gpu": 22.02, "tokens/trainable": 79032} +{"epoch": 0.68359375, "grad_norm": 0.16361771523952484, "learning_rate": 8.076468193435301e-05, "loss": 0.003869644831866026, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.64, "memory/max_allocated (GiB)": 36.64, "ppl": 1.00388, "step": 175, "tokens/total": 6045072, "tokens/train_per_sec_per_gpu": 25.72, "tokens/trainable": 79456} +{"epoch": 0.6875, "grad_norm": 0.48794642090797424, "learning_rate": 8.052614644612253e-05, "loss": 0.008433650247752666, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.02, "memory/max_allocated (GiB)": 39.02, "ppl": 1.00847, "step": 176, "tokens/total": 6082336, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 79917} +{"epoch": 0.69140625, "grad_norm": 0.36978980898857117, "learning_rate": 8.028654871074489e-05, "loss": 0.006972731091082096, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.007, "step": 177, "tokens/total": 6117024, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 80358} +{"epoch": 0.6953125, "grad_norm": 0.2405240833759308, "learning_rate": 8.004589869885986e-05, "loss": 0.005577913485467434, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.49, "memory/max_allocated (GiB)": 36.49, "ppl": 1.00559, "step": 178, "tokens/total": 6149136, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 80777} +{"epoch": 0.69921875, "grad_norm": 0.24249066412448883, "learning_rate": 7.980420642489674e-05, "loss": 0.003016636474058032, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.96, "memory/max_allocated (GiB)": 38.96, "ppl": 1.00302, "step": 179, "tokens/total": 6187776, "tokens/train_per_sec_per_gpu": 23.17, "tokens/trainable": 81250} +{"epoch": 0.703125, "grad_norm": 0.5789620280265808, "learning_rate": 7.95614819466576e-05, "loss": 0.0026860747020691633, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00269, "step": 180, "tokens/total": 6221728, "tokens/train_per_sec_per_gpu": 28.07, "tokens/trainable": 81715} +{"epoch": 0.70703125, "grad_norm": 1.1666159629821777, "learning_rate": 7.931773536489872e-05, "loss": 0.029250284656882286, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.02968, "step": 181, "tokens/total": 6255920, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 82160} +{"epoch": 0.7109375, "grad_norm": 0.8073496222496033, "learning_rate": 7.907297682291035e-05, "loss": 0.01186411827802658, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01193, "step": 182, "tokens/total": 6291104, "tokens/train_per_sec_per_gpu": 26.28, "tokens/trainable": 82624} +{"epoch": 0.71484375, "grad_norm": 0.52348393201828, "learning_rate": 7.882721650609442e-05, "loss": 0.008316706866025925, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00835, "step": 183, "tokens/total": 6330976, "tokens/train_per_sec_per_gpu": 23.01, "tokens/trainable": 83125} +{"epoch": 0.71875, "grad_norm": 1.4482166767120361, "learning_rate": 7.85804646415409e-05, "loss": 0.02300504967570305, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.02327, "step": 184, "tokens/total": 6366160, "tokens/train_per_sec_per_gpu": 26.54, "tokens/trainable": 83581} +{"epoch": 0.72265625, "grad_norm": 0.08995260298252106, "learning_rate": 7.833273149760207e-05, "loss": 0.001999420579522848, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.002, "step": 185, "tokens/total": 6397280, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 84051} +{"epoch": 0.7265625, "grad_norm": 1.3258529901504517, "learning_rate": 7.808402738346527e-05, "loss": 0.03866976127028465, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.03943, "step": 186, "tokens/total": 6435632, "tokens/train_per_sec_per_gpu": 23.21, "tokens/trainable": 84500} +{"epoch": 0.73046875, "grad_norm": 0.21925266087055206, "learning_rate": 7.783436264872382e-05, "loss": 0.003789495211094618, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.0038, "step": 187, "tokens/total": 6472752, "tokens/train_per_sec_per_gpu": 24.58, "tokens/trainable": 84986} +{"epoch": 0.734375, "grad_norm": 0.13683557510375977, "learning_rate": 7.758374768294647e-05, "loss": 0.004529173951596022, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.05, "memory/max_allocated (GiB)": 38.05, "ppl": 1.00454, "step": 188, "tokens/total": 6508528, "tokens/train_per_sec_per_gpu": 26.74, "tokens/trainable": 85465} +{"epoch": 0.73828125, "grad_norm": 0.4005732834339142, "learning_rate": 7.733219291524489e-05, "loss": 0.007882597856223583, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.59, "memory/max_allocated (GiB)": 37.59, "ppl": 1.00791, "step": 189, "tokens/total": 6542048, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 85896} +{"epoch": 0.7421875, "grad_norm": 3.5984132289886475, "learning_rate": 7.707970881383977e-05, "loss": 0.020785454660654068, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.57, "memory/max_allocated (GiB)": 36.57, "ppl": 1.021, "step": 190, "tokens/total": 6573792, "tokens/train_per_sec_per_gpu": 27.82, "tokens/trainable": 86346} +{"epoch": 0.74609375, "grad_norm": 0.3372122347354889, "learning_rate": 7.682630588562518e-05, "loss": 0.013578877784311771, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.01367, "step": 191, "tokens/total": 6608848, "tokens/train_per_sec_per_gpu": 24.11, "tokens/trainable": 86777} +{"epoch": 0.75, "grad_norm": 0.2558715045452118, "learning_rate": 7.657199467573129e-05, "loss": 0.00844060443341732, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.11, "memory/max_allocated (GiB)": 37.11, "ppl": 1.00848, "step": 192, "tokens/total": 6642560, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 87249} +{"epoch": 0.75390625, "grad_norm": 0.1345452070236206, "learning_rate": 7.631678576708561e-05, "loss": 0.002991447923704982, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.003, "step": 193, "tokens/total": 6676272, "tokens/train_per_sec_per_gpu": 28.73, "tokens/trainable": 87741} +{"epoch": 0.7578125, "grad_norm": 0.19830429553985596, "learning_rate": 7.606068977997255e-05, "loss": 0.00488736666738987, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.0049, "step": 194, "tokens/total": 6713568, "tokens/train_per_sec_per_gpu": 25.47, "tokens/trainable": 88219} +{"epoch": 0.76171875, "grad_norm": 0.611735463142395, "learning_rate": 7.580371737159148e-05, "loss": 0.00859236903488636, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00863, "step": 195, "tokens/total": 6749296, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 88630} +{"epoch": 0.765625, "grad_norm": 0.21242955327033997, "learning_rate": 7.554587923561324e-05, "loss": 0.004254139494150877, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00426, "step": 196, "tokens/total": 6783952, "tokens/train_per_sec_per_gpu": 24.42, "tokens/trainable": 89068} +{"epoch": 0.76953125, "grad_norm": 0.5875733494758606, "learning_rate": 7.528718610173511e-05, "loss": 0.017840035259723663, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.48, "memory/max_allocated (GiB)": 37.48, "ppl": 1.018, "step": 197, "tokens/total": 6819152, "tokens/train_per_sec_per_gpu": 22.61, "tokens/trainable": 89489} +{"epoch": 0.7734375, "grad_norm": 0.42628908157348633, "learning_rate": 7.502764873523431e-05, "loss": 0.007608545478433371, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00764, "step": 198, "tokens/total": 6853440, "tokens/train_per_sec_per_gpu": 27.01, "tokens/trainable": 89928} +{"epoch": 0.77734375, "grad_norm": 0.3049071431159973, "learning_rate": 7.476727793652011e-05, "loss": 0.005967305973172188, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.71, "memory/max_allocated (GiB)": 36.71, "ppl": 1.00599, "step": 199, "tokens/total": 6886272, "tokens/train_per_sec_per_gpu": 27.84, "tokens/trainable": 90407} +{"epoch": 0.78125, "grad_norm": 0.33770474791526794, "learning_rate": 7.450608454068415e-05, "loss": 0.012610118836164474, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.77, "memory/max_allocated (GiB)": 37.77, "ppl": 1.01269, "step": 200, "tokens/total": 6920624, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 90849} +{"epoch": 0.78515625, "grad_norm": 0.5401440858840942, "learning_rate": 7.424407941704987e-05, "loss": 0.00573667511343956, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00575, "step": 201, "tokens/total": 6957328, "tokens/train_per_sec_per_gpu": 27.05, "tokens/trainable": 91350} +{"epoch": 0.7890625, "grad_norm": 0.7998449802398682, "learning_rate": 7.398127346871986e-05, "loss": 0.020344752818346024, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.02055, "step": 202, "tokens/total": 6994048, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 91805} +{"epoch": 0.79296875, "grad_norm": 0.061208952218294144, "learning_rate": 7.371767763212238e-05, "loss": 0.001285267760977149, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00129, "step": 203, "tokens/total": 7029040, "tokens/train_per_sec_per_gpu": 26.31, "tokens/trainable": 92258} +{"epoch": 0.796875, "grad_norm": 0.5251902937889099, "learning_rate": 7.345330287655617e-05, "loss": 0.007642202079296112, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00767, "step": 204, "tokens/total": 7067104, "tokens/train_per_sec_per_gpu": 22.51, "tokens/trainable": 92721} +{"epoch": 0.80078125, "grad_norm": 0.5568168759346008, "learning_rate": 7.31881602037339e-05, "loss": 0.018840672448277473, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.01902, "step": 205, "tokens/total": 7099136, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 93160} +{"epoch": 0.8046875, "grad_norm": 0.8956170082092285, "learning_rate": 7.29222606473245e-05, "loss": 0.01183111872524023, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.33, "memory/max_allocated (GiB)": 37.33, "ppl": 1.0119, "step": 206, "tokens/total": 7132208, "tokens/train_per_sec_per_gpu": 26.22, "tokens/trainable": 93605} +{"epoch": 0.80859375, "grad_norm": 0.2926824390888214, "learning_rate": 7.265561527249383e-05, "loss": 0.007752003613859415, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00778, "step": 207, "tokens/total": 7166256, "tokens/train_per_sec_per_gpu": 27.4, "tokens/trainable": 94041} +{"epoch": 0.8125, "grad_norm": 0.38738927245140076, "learning_rate": 7.238823517544436e-05, "loss": 0.011301856487989426, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.01137, "step": 208, "tokens/total": 7203152, "tokens/train_per_sec_per_gpu": 28.02, "tokens/trainable": 94512} +{"epoch": 0.81640625, "grad_norm": 0.10928687453269958, "learning_rate": 7.212013148295333e-05, "loss": 0.002069462789222598, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.38, "memory/max_allocated (GiB)": 37.38, "ppl": 1.00207, "step": 209, "tokens/total": 7237264, "tokens/train_per_sec_per_gpu": 29.53, "tokens/trainable": 94994} +{"epoch": 0.8203125, "grad_norm": 0.23567059636116028, "learning_rate": 7.185131535190975e-05, "loss": 0.001254252390936017, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00126, "step": 210, "tokens/total": 7269664, "tokens/train_per_sec_per_gpu": 29.72, "tokens/trainable": 95503} +{"epoch": 0.82421875, "grad_norm": 0.263143926858902, "learning_rate": 7.158179796885005e-05, "loss": 0.0074745831079781055, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.0075, "step": 211, "tokens/total": 7304080, "tokens/train_per_sec_per_gpu": 27.27, "tokens/trainable": 95954} +{"epoch": 0.828125, "grad_norm": 0.36440420150756836, "learning_rate": 7.131159054949273e-05, "loss": 0.006694465409964323, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00672, "step": 212, "tokens/total": 7342480, "tokens/train_per_sec_per_gpu": 22.15, "tokens/trainable": 96428} +{"epoch": 0.83203125, "grad_norm": 0.5311977863311768, "learning_rate": 7.104070433827139e-05, "loss": 0.01318358164280653, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.01327, "step": 213, "tokens/total": 7378032, "tokens/train_per_sec_per_gpu": 22.85, "tokens/trainable": 96828} +{"epoch": 0.8359375, "grad_norm": 0.22275257110595703, "learning_rate": 7.076915060786705e-05, "loss": 0.004414305090904236, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00442, "step": 214, "tokens/total": 7410928, "tokens/train_per_sec_per_gpu": 24.04, "tokens/trainable": 97270} +{"epoch": 0.83984375, "grad_norm": 0.5393062829971313, "learning_rate": 7.049694065873882e-05, "loss": 0.007394821383059025, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00742, "step": 215, "tokens/total": 7445664, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 97727} +{"epoch": 0.84375, "grad_norm": 0.5993046760559082, "learning_rate": 7.022408581865382e-05, "loss": 0.012309437617659569, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.01239, "step": 216, "tokens/total": 7479984, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 98201} +{"epoch": 0.84765625, "grad_norm": 0.07589630037546158, "learning_rate": 6.99505974422157e-05, "loss": 0.0007467762334272265, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.65, "memory/max_allocated (GiB)": 37.65, "ppl": 1.00075, "step": 217, "tokens/total": 7514144, "tokens/train_per_sec_per_gpu": 28.14, "tokens/trainable": 98674} +{"epoch": 0.8515625, "grad_norm": 0.05518501624464989, "learning_rate": 6.967648691039213e-05, "loss": 0.00044069692376069725, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.73, "memory/max_allocated (GiB)": 36.73, "ppl": 1.00044, "step": 218, "tokens/total": 7546256, "tokens/train_per_sec_per_gpu": 27.69, "tokens/trainable": 99108} +{"epoch": 0.85546875, "grad_norm": 0.13123318552970886, "learning_rate": 6.940176563004123e-05, "loss": 0.0024234973825514317, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00243, "step": 219, "tokens/total": 7580432, "tokens/train_per_sec_per_gpu": 25.75, "tokens/trainable": 99540} +{"epoch": 0.859375, "grad_norm": 0.31413328647613525, "learning_rate": 6.912644503343682e-05, "loss": 0.0018550190143287182, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.00186, "step": 220, "tokens/total": 7617904, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 99978} +{"epoch": 0.86328125, "grad_norm": 0.8278009295463562, "learning_rate": 6.885053657779273e-05, "loss": 0.013766925781965256, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.01386, "step": 221, "tokens/total": 7649696, "tokens/train_per_sec_per_gpu": 28.12, "tokens/trainable": 100450} +{"epoch": 0.8671875, "grad_norm": 0.09925957769155502, "learning_rate": 6.857405174478604e-05, "loss": 0.0009696983033791184, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.32, "memory/max_allocated (GiB)": 37.32, "ppl": 1.00097, "step": 222, "tokens/total": 7684464, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 100903} +{"epoch": 0.87109375, "grad_norm": 0.4999784827232361, "learning_rate": 6.82970020400792e-05, "loss": 0.02127908356487751, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02151, "step": 223, "tokens/total": 7721936, "tokens/train_per_sec_per_gpu": 22.14, "tokens/trainable": 101351} +{"epoch": 0.875, "grad_norm": 0.754840075969696, "learning_rate": 6.801939899284132e-05, "loss": 0.009632373228669167, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00968, "step": 224, "tokens/total": 7758160, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 101857} +{"epoch": 0.87890625, "grad_norm": 0.1402512788772583, "learning_rate": 6.774125415526827e-05, "loss": 0.0016685896553099155, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00167, "step": 225, "tokens/total": 7791216, "tokens/train_per_sec_per_gpu": 27.52, "tokens/trainable": 102328} +{"epoch": 0.8828125, "grad_norm": 0.18280313909053802, "learning_rate": 6.746257910210214e-05, "loss": 0.0032010748982429504, "memory/device_reserved (GiB)": 43.25, "memory/max_active (GiB)": 37.12, "memory/max_allocated (GiB)": 37.12, "ppl": 1.00321, "step": 226, "tokens/total": 7824400, "tokens/train_per_sec_per_gpu": 27.9, "tokens/trainable": 102779} +{"epoch": 0.88671875, "grad_norm": 0.2275019884109497, "learning_rate": 6.718338543014937e-05, "loss": 0.0030386706348508596, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 35.75, "memory/max_allocated (GiB)": 35.75, "ppl": 1.00304, "step": 227, "tokens/total": 7854672, "tokens/train_per_sec_per_gpu": 30.6, "tokens/trainable": 103228} +{"epoch": 0.890625, "grad_norm": 0.43627986311912537, "learning_rate": 6.69036847577983e-05, "loss": 0.004438947886228561, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.12, "memory/max_allocated (GiB)": 38.12, "ppl": 1.00445, "step": 228, "tokens/total": 7888592, "tokens/train_per_sec_per_gpu": 26.49, "tokens/trainable": 103684} +{"epoch": 0.89453125, "grad_norm": 0.4158158600330353, "learning_rate": 6.662348872453553e-05, "loss": 0.006800580304116011, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00682, "step": 229, "tokens/total": 7924848, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 104145} +{"epoch": 0.8984375, "grad_norm": 0.2914593517780304, "learning_rate": 6.63428089904618e-05, "loss": 0.0046365936286747456, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00465, "step": 230, "tokens/total": 7963856, "tokens/train_per_sec_per_gpu": 23.99, "tokens/trainable": 104598} +{"epoch": 0.90234375, "grad_norm": 0.40541577339172363, "learning_rate": 6.60616572358065e-05, "loss": 0.0032450316939502954, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.21, "memory/max_allocated (GiB)": 37.21, "ppl": 1.00325, "step": 231, "tokens/total": 7998320, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 104998} +{"epoch": 0.90625, "grad_norm": 0.21683749556541443, "learning_rate": 6.578004516044172e-05, "loss": 0.0024478500708937645, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00245, "step": 232, "tokens/total": 8034096, "tokens/train_per_sec_per_gpu": 27.31, "tokens/trainable": 105465} +{"epoch": 0.91015625, "grad_norm": 0.5367693305015564, "learning_rate": 6.549798448339548e-05, "loss": 0.007329205051064491, "memory/device_reserved (GiB)": 43.26, "memory/max_active (GiB)": 38.59, "memory/max_allocated (GiB)": 38.59, "ppl": 1.00736, "step": 233, "tokens/total": 8071360, "tokens/train_per_sec_per_gpu": 24.77, "tokens/trainable": 105928} +{"epoch": 0.9140625, "grad_norm": 0.07326225936412811, "learning_rate": 6.521548694236384e-05, "loss": 0.0006970649701543152, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.0007, "step": 234, "tokens/total": 8108560, "tokens/train_per_sec_per_gpu": 23.13, "tokens/trainable": 106365} +{"epoch": 0.91796875, "grad_norm": 0.9467607140541077, "learning_rate": 6.493256429322259e-05, "loss": 0.01661212183535099, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.01675, "step": 235, "tokens/total": 8144816, "tokens/train_per_sec_per_gpu": 21.99, "tokens/trainable": 106796} +{"epoch": 0.921875, "grad_norm": 0.7784695625305176, "learning_rate": 6.464922830953799e-05, "loss": 0.029151076450943947, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.02958, "step": 236, "tokens/total": 8183680, "tokens/train_per_sec_per_gpu": 22.08, "tokens/trainable": 107227} +{"epoch": 0.92578125, "grad_norm": 1.6428377628326416, "learning_rate": 6.436549078207688e-05, "loss": 0.03603469952940941, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.03669, "step": 237, "tokens/total": 8216688, "tokens/train_per_sec_per_gpu": 28.46, "tokens/trainable": 107687} +{"epoch": 0.9296875, "grad_norm": 0.8423205614089966, "learning_rate": 6.408136351831592e-05, "loss": 0.020360495895147324, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.02057, "step": 238, "tokens/total": 8249104, "tokens/train_per_sec_per_gpu": 23.84, "tokens/trainable": 108127} +{"epoch": 0.93359375, "grad_norm": 0.3961838483810425, "learning_rate": 6.379685834195036e-05, "loss": 0.007839069701731205, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00787, "step": 239, "tokens/total": 8288528, "tokens/train_per_sec_per_gpu": 24.27, "tokens/trainable": 108633} +{"epoch": 0.9375, "grad_norm": 0.355879545211792, "learning_rate": 6.351198709240186e-05, "loss": 0.005410174373537302, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.44, "memory/max_allocated (GiB)": 39.44, "ppl": 1.00542, "step": 240, "tokens/total": 8328464, "tokens/train_per_sec_per_gpu": 20.79, "tokens/trainable": 109085} +{"epoch": 0.94140625, "grad_norm": 0.3391614258289337, "learning_rate": 6.32267616243259e-05, "loss": 0.008839430287480354, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00888, "step": 241, "tokens/total": 8361264, "tokens/train_per_sec_per_gpu": 28.39, "tokens/trainable": 109548} +{"epoch": 0.9453125, "grad_norm": 0.354306161403656, "learning_rate": 6.294119380711849e-05, "loss": 0.008945208042860031, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.39, "memory/max_allocated (GiB)": 38.39, "ppl": 1.00899, "step": 242, "tokens/total": 8395552, "tokens/train_per_sec_per_gpu": 23.54, "tokens/trainable": 109975} +{"epoch": 0.94921875, "grad_norm": 0.5887778401374817, "learning_rate": 6.265529552442209e-05, "loss": 0.01206921972334385, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.69, "memory/max_allocated (GiB)": 37.69, "ppl": 1.01214, "step": 243, "tokens/total": 8430656, "tokens/train_per_sec_per_gpu": 27.71, "tokens/trainable": 110436} +{"epoch": 0.953125, "grad_norm": 1.3057684898376465, "learning_rate": 6.236907867363127e-05, "loss": 0.015023745596408844, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.17, "memory/max_allocated (GiB)": 38.17, "ppl": 1.01514, "step": 244, "tokens/total": 8465232, "tokens/train_per_sec_per_gpu": 26.3, "tokens/trainable": 110882} +{"epoch": 0.95703125, "grad_norm": 0.1653934121131897, "learning_rate": 6.208255516539749e-05, "loss": 0.0024531311355531216, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.00246, "step": 245, "tokens/total": 8497552, "tokens/train_per_sec_per_gpu": 26.26, "tokens/trainable": 111355} +{"epoch": 0.9609375, "grad_norm": 0.1979655772447586, "learning_rate": 6.179573692313344e-05, "loss": 0.0037991702556610107, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.8, "memory/max_allocated (GiB)": 38.8, "ppl": 1.00381, "step": 246, "tokens/total": 8534336, "tokens/train_per_sec_per_gpu": 25.73, "tokens/trainable": 111803} +{"epoch": 0.96484375, "grad_norm": 0.2744027078151703, "learning_rate": 6.150863588251694e-05, "loss": 0.006413429509848356, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00643, "step": 247, "tokens/total": 8574224, "tokens/train_per_sec_per_gpu": 22.41, "tokens/trainable": 112270} +{"epoch": 0.96875, "grad_norm": 0.7485118508338928, "learning_rate": 6.122126399099419e-05, "loss": 0.016094401478767395, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.01622, "step": 248, "tokens/total": 8613616, "tokens/train_per_sec_per_gpu": 26.51, "tokens/trainable": 112778} +{"epoch": 0.97265625, "grad_norm": 0.5279822945594788, "learning_rate": 6.0933633207282615e-05, "loss": 0.029549092054367065, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.02999, "step": 249, "tokens/total": 8650048, "tokens/train_per_sec_per_gpu": 26.96, "tokens/trainable": 113284} +{"epoch": 0.9765625, "grad_norm": 0.6191792488098145, "learning_rate": 6.064575550087316e-05, "loss": 0.013943398371338844, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01404, "step": 250, "tokens/total": 8686688, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 113722} +{"epoch": 0.98046875, "grad_norm": 0.3184222877025604, "learning_rate": 6.0357642851532245e-05, "loss": 0.009820410050451756, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00987, "step": 251, "tokens/total": 8723376, "tokens/train_per_sec_per_gpu": 25.46, "tokens/trainable": 114213} +{"epoch": 0.984375, "grad_norm": 1.0236104726791382, "learning_rate": 6.0069307248803294e-05, "loss": 0.01883062906563282, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.01901, "step": 252, "tokens/total": 8758736, "tokens/train_per_sec_per_gpu": 21.65, "tokens/trainable": 114634} +{"epoch": 0.98828125, "grad_norm": 0.34827399253845215, "learning_rate": 5.9780760691507635e-05, "loss": 0.01312404777854681, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.01321, "step": 253, "tokens/total": 8790480, "tokens/train_per_sec_per_gpu": 26.38, "tokens/trainable": 115066} +{"epoch": 0.9921875, "grad_norm": 0.253101110458374, "learning_rate": 5.9492015187245334e-05, "loss": 0.006610496900975704, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00663, "step": 254, "tokens/total": 8828224, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 115520} +{"epoch": 0.99609375, "grad_norm": 0.2707793414592743, "learning_rate": 5.920308275189541e-05, "loss": 0.005402787588536739, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 36.8, "memory/max_allocated (GiB)": 36.8, "ppl": 1.00542, "step": 255, "tokens/total": 8860400, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 115921} +{"epoch": 1.0, "grad_norm": 0.4474392831325531, "learning_rate": 5.8913975409115874e-05, "loss": 0.010412642732262611, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.01047, "step": 256, "tokens/total": 8893760, "tokens/train_per_sec_per_gpu": 25.03, "tokens/trainable": 116384} +{"epoch": 1.00390625, "grad_norm": 0.35948696732521057, "learning_rate": 5.8624705189843395e-05, "loss": 0.007576838601380587, "memory/device_reserved (GiB)": 43.65, "memory/max_active (GiB)": 38.95, "memory/max_allocated (GiB)": 38.95, "ppl": 1.00761, "step": 257, "tokens/total": 8930976, "tokens/train_per_sec_per_gpu": 24.37, "tokens/trainable": 116871} +{"epoch": 1.0078125, "grad_norm": 0.06467556953430176, "learning_rate": 5.833528413179249e-05, "loss": 0.0012656881008297205, "memory/device_reserved (GiB)": 42.25, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00127, "step": 258, "tokens/total": 8967472, "tokens/train_per_sec_per_gpu": 27.48, "tokens/trainable": 117369} +{"epoch": 1.01171875, "grad_norm": 0.3001314699649811, "learning_rate": 5.80457242789548e-05, "loss": 0.0036677327007055283, "memory/device_reserved (GiB)": 42.31, "memory/max_active (GiB)": 39.47, "memory/max_allocated (GiB)": 39.47, "ppl": 1.00367, "step": 259, "tokens/total": 9007456, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 117866} +{"epoch": 1.015625, "grad_norm": 0.5105038285255432, "learning_rate": 5.77560376810977e-05, "loss": 0.004875686019659042, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00489, "step": 260, "tokens/total": 9041104, "tokens/train_per_sec_per_gpu": 28.82, "tokens/trainable": 118330} +{"epoch": 1.01953125, "grad_norm": 0.4467502534389496, "learning_rate": 5.7466236393263005e-05, "loss": 0.013924190774559975, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.13, "memory/max_allocated (GiB)": 39.13, "ppl": 1.01402, "step": 261, "tokens/total": 9076176, "tokens/train_per_sec_per_gpu": 25.2, "tokens/trainable": 118776} +{"epoch": 1.0234375, "grad_norm": 0.3719651997089386, "learning_rate": 5.717633247526522e-05, "loss": 0.0067051975056529045, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00673, "step": 262, "tokens/total": 9107472, "tokens/train_per_sec_per_gpu": 27.34, "tokens/trainable": 119211} +{"epoch": 1.02734375, "grad_norm": 1.0953550338745117, "learning_rate": 5.688633799118971e-05, "loss": 0.0009893823880702257, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.86, "memory/max_allocated (GiB)": 36.86, "ppl": 1.00099, "step": 263, "tokens/total": 9139552, "tokens/train_per_sec_per_gpu": 23.11, "tokens/trainable": 119616} +{"epoch": 1.03125, "grad_norm": 0.5340035557746887, "learning_rate": 5.659626500889066e-05, "loss": 0.011958985589444637, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.01203, "step": 264, "tokens/total": 9173424, "tokens/train_per_sec_per_gpu": 25.5, "tokens/trainable": 120086} +{"epoch": 1.03515625, "grad_norm": 0.051546838134527206, "learning_rate": 5.6306125599488905e-05, "loss": 0.00087040348444134, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00087, "step": 265, "tokens/total": 9203984, "tokens/train_per_sec_per_gpu": 31.75, "tokens/trainable": 120525} +{"epoch": 1.0390625, "grad_norm": 0.41811415553092957, "learning_rate": 5.601593183686955e-05, "loss": 0.006137186661362648, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.69, "memory/max_allocated (GiB)": 38.69, "ppl": 1.00616, "step": 266, "tokens/total": 9241248, "tokens/train_per_sec_per_gpu": 25.65, "tokens/trainable": 121001} +{"epoch": 1.04296875, "grad_norm": 0.38518908619880676, "learning_rate": 5.572569579717961e-05, "loss": 0.006210450083017349, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.18, "memory/max_allocated (GiB)": 39.18, "ppl": 1.00623, "step": 267, "tokens/total": 9276768, "tokens/train_per_sec_per_gpu": 24.16, "tokens/trainable": 121465} +{"epoch": 1.046875, "grad_norm": 0.5231328010559082, "learning_rate": 5.543542955832538e-05, "loss": 0.002358397701755166, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00236, "step": 268, "tokens/total": 9316528, "tokens/train_per_sec_per_gpu": 21.41, "tokens/trainable": 121908} +{"epoch": 1.05078125, "grad_norm": 0.24940060079097748, "learning_rate": 5.514514519946986e-05, "loss": 0.004307364579290152, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.31, "memory/max_allocated (GiB)": 39.31, "ppl": 1.00432, "step": 269, "tokens/total": 9356352, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 122406} +{"epoch": 1.0546875, "grad_norm": 0.21064507961273193, "learning_rate": 5.485485480053015e-05, "loss": 0.0024004678707569838, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.51, "memory/max_allocated (GiB)": 39.51, "ppl": 1.0024, "step": 270, "tokens/total": 9394576, "tokens/train_per_sec_per_gpu": 21.59, "tokens/trainable": 122812} +{"epoch": 1.05859375, "grad_norm": 0.4254460632801056, "learning_rate": 5.4564570441674645e-05, "loss": 0.0054376740008592606, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.92, "memory/max_allocated (GiB)": 36.92, "ppl": 1.00545, "step": 271, "tokens/total": 9427840, "tokens/train_per_sec_per_gpu": 25.18, "tokens/trainable": 123252} +{"epoch": 1.0625, "grad_norm": 0.10426627099514008, "learning_rate": 5.42743042028204e-05, "loss": 0.0011718064779415727, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.00117, "step": 272, "tokens/total": 9463536, "tokens/train_per_sec_per_gpu": 20.36, "tokens/trainable": 123692} +{"epoch": 1.06640625, "grad_norm": 0.6593073010444641, "learning_rate": 5.3984068163130464e-05, "loss": 0.012782225385308266, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.09, "memory/max_allocated (GiB)": 37.09, "ppl": 1.01286, "step": 273, "tokens/total": 9497248, "tokens/train_per_sec_per_gpu": 27.78, "tokens/trainable": 124162} +{"epoch": 1.0703125, "grad_norm": 0.397914320230484, "learning_rate": 5.369387440051111e-05, "loss": 0.005888782441616058, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00591, "step": 274, "tokens/total": 9530784, "tokens/train_per_sec_per_gpu": 25.12, "tokens/trainable": 124625} +{"epoch": 1.07421875, "grad_norm": 0.028597824275493622, "learning_rate": 5.340373499110935e-05, "loss": 0.00022801666636951268, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00023, "step": 275, "tokens/total": 9566656, "tokens/train_per_sec_per_gpu": 26.19, "tokens/trainable": 125119} +{"epoch": 1.078125, "grad_norm": 0.08383466303348541, "learning_rate": 5.3113662008810304e-05, "loss": 0.0005590343498624861, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00056, "step": 276, "tokens/total": 9606672, "tokens/train_per_sec_per_gpu": 24.7, "tokens/trainable": 125601} +{"epoch": 1.08203125, "grad_norm": 0.534981369972229, "learning_rate": 5.282366752473479e-05, "loss": 0.007298499811440706, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.23, "memory/max_allocated (GiB)": 38.23, "ppl": 1.00733, "step": 277, "tokens/total": 9643680, "tokens/train_per_sec_per_gpu": 24.21, "tokens/trainable": 126075} +{"epoch": 1.0859375, "grad_norm": 0.686750054359436, "learning_rate": 5.2533763606737005e-05, "loss": 0.01284632459282875, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.01293, "step": 278, "tokens/total": 9683200, "tokens/train_per_sec_per_gpu": 19.14, "tokens/trainable": 126491} +{"epoch": 1.08984375, "grad_norm": 0.34263017773628235, "learning_rate": 5.224396231890232e-05, "loss": 0.0038576049264520407, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00387, "step": 279, "tokens/total": 9714672, "tokens/train_per_sec_per_gpu": 29.7, "tokens/trainable": 126940} +{"epoch": 1.09375, "grad_norm": 0.5330859422683716, "learning_rate": 5.195427572104522e-05, "loss": 0.01094902865588665, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.01101, "step": 280, "tokens/total": 9749104, "tokens/train_per_sec_per_gpu": 23.49, "tokens/trainable": 127376} +{"epoch": 1.09765625, "grad_norm": 0.15562763810157776, "learning_rate": 5.166471586820751e-05, "loss": 0.0016295767854899168, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.35, "memory/max_allocated (GiB)": 39.35, "ppl": 1.00163, "step": 281, "tokens/total": 9785712, "tokens/train_per_sec_per_gpu": 23.53, "tokens/trainable": 127841} +{"epoch": 1.1015625, "grad_norm": 0.3615416884422302, "learning_rate": 5.1375294810156615e-05, "loss": 0.014125174842774868, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.01423, "step": 282, "tokens/total": 9820544, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 128288} +{"epoch": 1.10546875, "grad_norm": 0.21486680209636688, "learning_rate": 5.1086024590884144e-05, "loss": 0.0016033012652769685, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.0016, "step": 283, "tokens/total": 9857952, "tokens/train_per_sec_per_gpu": 24.65, "tokens/trainable": 128737} +{"epoch": 1.109375, "grad_norm": 0.3502711057662964, "learning_rate": 5.079691724810461e-05, "loss": 0.003668692195788026, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.33, "memory/max_allocated (GiB)": 38.33, "ppl": 1.00368, "step": 284, "tokens/total": 9894128, "tokens/train_per_sec_per_gpu": 21.54, "tokens/trainable": 129172} +{"epoch": 1.11328125, "grad_norm": 0.1279154121875763, "learning_rate": 5.0507984812754684e-05, "loss": 0.0010889992117881775, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.17, "memory/max_allocated (GiB)": 37.17, "ppl": 1.00109, "step": 285, "tokens/total": 9927568, "tokens/train_per_sec_per_gpu": 28.48, "tokens/trainable": 129669} +{"epoch": 1.1171875, "grad_norm": 0.6582543849945068, "learning_rate": 5.021923930849237e-05, "loss": 0.016363991424441338, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.42, "memory/max_allocated (GiB)": 37.42, "ppl": 1.0165, "step": 286, "tokens/total": 9961888, "tokens/train_per_sec_per_gpu": 27.67, "tokens/trainable": 130155} +{"epoch": 1.12109375, "grad_norm": 0.13749751448631287, "learning_rate": 4.99306927511967e-05, "loss": 0.0024362634867429733, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.31, "memory/max_allocated (GiB)": 37.31, "ppl": 1.00244, "step": 287, "tokens/total": 9995696, "tokens/train_per_sec_per_gpu": 27.6, "tokens/trainable": 130595} +{"epoch": 1.125, "grad_norm": 0.2623278498649597, "learning_rate": 4.964235714846775e-05, "loss": 0.008338609710335732, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.72, "memory/max_allocated (GiB)": 36.72, "ppl": 1.00837, "step": 288, "tokens/total": 10028000, "tokens/train_per_sec_per_gpu": 26.89, "tokens/trainable": 131015} +{"epoch": 1.12890625, "grad_norm": 0.11188921332359314, "learning_rate": 4.9354244499126866e-05, "loss": 0.0010074015008285642, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 36.16, "memory/max_allocated (GiB)": 36.16, "ppl": 1.00101, "step": 289, "tokens/total": 10059728, "tokens/train_per_sec_per_gpu": 28.98, "tokens/trainable": 131481} +{"epoch": 1.1328125, "grad_norm": 0.12944196164608002, "learning_rate": 4.90663667927174e-05, "loss": 0.0019044274231418967, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00191, "step": 290, "tokens/total": 10092576, "tokens/train_per_sec_per_gpu": 28.44, "tokens/trainable": 131930} +{"epoch": 1.13671875, "grad_norm": 0.06529365479946136, "learning_rate": 4.877873600900581e-05, "loss": 0.0007035612943582237, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.99, "memory/max_allocated (GiB)": 37.99, "ppl": 1.0007, "step": 291, "tokens/total": 10128608, "tokens/train_per_sec_per_gpu": 21.43, "tokens/trainable": 132388} +{"epoch": 1.140625, "grad_norm": 0.40535783767700195, "learning_rate": 4.849136411748306e-05, "loss": 0.0039036402013152838, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00391, "step": 292, "tokens/total": 10166192, "tokens/train_per_sec_per_gpu": 24.63, "tokens/trainable": 132860} +{"epoch": 1.14453125, "grad_norm": 0.22061260044574738, "learning_rate": 4.8204263076866574e-05, "loss": 0.007592522539198399, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00762, "step": 293, "tokens/total": 10201840, "tokens/train_per_sec_per_gpu": 29.3, "tokens/trainable": 133360} +{"epoch": 1.1484375, "grad_norm": 0.7864055633544922, "learning_rate": 4.791744483460251e-05, "loss": 0.010605808347463608, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.01066, "step": 294, "tokens/total": 10236048, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 133793} +{"epoch": 1.15234375, "grad_norm": 0.11420025676488876, "learning_rate": 4.7630921326368736e-05, "loss": 0.002142795128747821, "memory/device_reserved (GiB)": 42.33, "memory/max_active (GiB)": 39.22, "memory/max_allocated (GiB)": 39.22, "ppl": 1.00215, "step": 295, "tokens/total": 10274688, "tokens/train_per_sec_per_gpu": 19.83, "tokens/trainable": 134218} +{"epoch": 1.15625, "grad_norm": 0.2983825206756592, "learning_rate": 4.7344704475577916e-05, "loss": 0.0023779014591127634, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.66, "memory/max_allocated (GiB)": 35.66, "ppl": 1.00238, "step": 296, "tokens/total": 10304576, "tokens/train_per_sec_per_gpu": 29.99, "tokens/trainable": 134654} +{"epoch": 1.16015625, "grad_norm": 0.7146530151367188, "learning_rate": 4.705880619288153e-05, "loss": 0.012010451406240463, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.21, "memory/max_allocated (GiB)": 38.21, "ppl": 1.01208, "step": 297, "tokens/total": 10340720, "tokens/train_per_sec_per_gpu": 21.81, "tokens/trainable": 135056} +{"epoch": 1.1640625, "grad_norm": 0.37344488501548767, "learning_rate": 4.677323837567412e-05, "loss": 0.0034731850028038025, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00348, "step": 298, "tokens/total": 10373376, "tokens/train_per_sec_per_gpu": 25.11, "tokens/trainable": 135483} +{"epoch": 1.16796875, "grad_norm": 0.10712098330259323, "learning_rate": 4.6488012907598146e-05, "loss": 0.0014444587286561728, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00145, "step": 299, "tokens/total": 10406128, "tokens/train_per_sec_per_gpu": 27.72, "tokens/trainable": 135957} +{"epoch": 1.171875, "grad_norm": 0.25059974193573, "learning_rate": 4.620314165804964e-05, "loss": 0.003132876008749008, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.3, "memory/max_allocated (GiB)": 38.3, "ppl": 1.00314, "step": 300, "tokens/total": 10441792, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 136430} +{"epoch": 1.17578125, "grad_norm": 0.0701121911406517, "learning_rate": 4.591863648168407e-05, "loss": 0.0012382074492052197, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.78, "memory/max_allocated (GiB)": 38.78, "ppl": 1.00124, "step": 301, "tokens/total": 10476128, "tokens/train_per_sec_per_gpu": 26.14, "tokens/trainable": 136896} +{"epoch": 1.1796875, "grad_norm": 0.02574644796550274, "learning_rate": 4.5634509217923135e-05, "loss": 0.000529083888977766, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00053, "step": 302, "tokens/total": 10509472, "tokens/train_per_sec_per_gpu": 25.56, "tokens/trainable": 137331} +{"epoch": 1.18359375, "grad_norm": 0.23367881774902344, "learning_rate": 4.535077169046201e-05, "loss": 0.0015380993718281388, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00154, "step": 303, "tokens/total": 10546032, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 137768} +{"epoch": 1.1875, "grad_norm": 1.0631389617919922, "learning_rate": 4.506743570677743e-05, "loss": 0.006021319888532162, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.38, "memory/max_allocated (GiB)": 36.38, "ppl": 1.00604, "step": 304, "tokens/total": 10576624, "tokens/train_per_sec_per_gpu": 31.06, "tokens/trainable": 138200} +{"epoch": 1.19140625, "grad_norm": 0.2094041109085083, "learning_rate": 4.478451305763618e-05, "loss": 0.003879548516124487, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.95, "memory/max_allocated (GiB)": 36.95, "ppl": 1.00389, "step": 305, "tokens/total": 10609760, "tokens/train_per_sec_per_gpu": 30.32, "tokens/trainable": 138660} +{"epoch": 1.1953125, "grad_norm": 0.037139009684324265, "learning_rate": 4.450201551660454e-05, "loss": 0.000537746527697891, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.51, "memory/max_allocated (GiB)": 36.51, "ppl": 1.00054, "step": 306, "tokens/total": 10641472, "tokens/train_per_sec_per_gpu": 26.02, "tokens/trainable": 139092} +{"epoch": 1.19921875, "grad_norm": 0.47949373722076416, "learning_rate": 4.4219954839558276e-05, "loss": 0.00670614093542099, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00673, "step": 307, "tokens/total": 10676160, "tokens/train_per_sec_per_gpu": 25.1, "tokens/trainable": 139533} +{"epoch": 1.203125, "grad_norm": 0.25365734100341797, "learning_rate": 4.393834276419352e-05, "loss": 0.0032138745300471783, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.83, "memory/max_allocated (GiB)": 35.83, "ppl": 1.00322, "step": 308, "tokens/total": 10706384, "tokens/train_per_sec_per_gpu": 31.84, "tokens/trainable": 140020} +{"epoch": 1.20703125, "grad_norm": 0.10202344506978989, "learning_rate": 4.36571910095382e-05, "loss": 0.0009595949668437243, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.9, "memory/max_allocated (GiB)": 38.9, "ppl": 1.00096, "step": 309, "tokens/total": 10744496, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 140465} +{"epoch": 1.2109375, "grad_norm": 0.13297803699970245, "learning_rate": 4.337651127546448e-05, "loss": 0.001210046000778675, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00121, "step": 310, "tokens/total": 10780176, "tokens/train_per_sec_per_gpu": 27.33, "tokens/trainable": 140915} +{"epoch": 1.21484375, "grad_norm": 0.4257793128490448, "learning_rate": 4.3096315242201736e-05, "loss": 0.004492432344704866, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.61, "memory/max_allocated (GiB)": 36.61, "ppl": 1.0045, "step": 311, "tokens/total": 10813136, "tokens/train_per_sec_per_gpu": 26.04, "tokens/trainable": 141350} +{"epoch": 1.21875, "grad_norm": 1.6273378133773804, "learning_rate": 4.2816614569850635e-05, "loss": 0.002729202387854457, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00273, "step": 312, "tokens/total": 10848912, "tokens/train_per_sec_per_gpu": 24.09, "tokens/trainable": 141789} +{"epoch": 1.22265625, "grad_norm": 0.20658275485038757, "learning_rate": 4.2537420897897864e-05, "loss": 0.002938120625913143, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00294, "step": 313, "tokens/total": 10884320, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 142266} +{"epoch": 1.2265625, "grad_norm": 0.06595562398433685, "learning_rate": 4.225874584473174e-05, "loss": 0.00040870148222893476, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.64, "memory/max_allocated (GiB)": 37.64, "ppl": 1.00041, "step": 314, "tokens/total": 10917152, "tokens/train_per_sec_per_gpu": 25.77, "tokens/trainable": 142704} +{"epoch": 1.23046875, "grad_norm": 0.15524443984031677, "learning_rate": 4.19806010071587e-05, "loss": 0.0012571783736348152, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.48, "memory/max_allocated (GiB)": 38.48, "ppl": 1.00126, "step": 315, "tokens/total": 10955024, "tokens/train_per_sec_per_gpu": 24.13, "tokens/trainable": 143168} +{"epoch": 1.234375, "grad_norm": 0.4104042053222656, "learning_rate": 4.170299795992081e-05, "loss": 0.009339426644146442, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00938, "step": 316, "tokens/total": 10988416, "tokens/train_per_sec_per_gpu": 26.7, "tokens/trainable": 143615} +{"epoch": 1.23828125, "grad_norm": 0.07205026596784592, "learning_rate": 4.142594825521398e-05, "loss": 0.0003858778509311378, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.33, "memory/max_allocated (GiB)": 39.33, "ppl": 1.00039, "step": 317, "tokens/total": 11027712, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 144096} +{"epoch": 1.2421875, "grad_norm": 0.596966028213501, "learning_rate": 4.114946342220728e-05, "loss": 0.0014998784754425287, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.82, "memory/max_allocated (GiB)": 38.82, "ppl": 1.0015, "step": 318, "tokens/total": 11063920, "tokens/train_per_sec_per_gpu": 23.66, "tokens/trainable": 144543} +{"epoch": 1.24609375, "grad_norm": 0.34074369072914124, "learning_rate": 4.087355496656321e-05, "loss": 0.005284292623400688, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.0053, "step": 319, "tokens/total": 11097952, "tokens/train_per_sec_per_gpu": 28.24, "tokens/trainable": 145016} +{"epoch": 1.25, "grad_norm": 0.4164718985557556, "learning_rate": 4.05982343699588e-05, "loss": 0.0041737607680261135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00418, "step": 320, "tokens/total": 11137168, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 145511} +{"epoch": 1.25390625, "grad_norm": 0.01693602092564106, "learning_rate": 4.0323513089607876e-05, "loss": 0.00016803256585262716, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00017, "step": 321, "tokens/total": 11172224, "tokens/train_per_sec_per_gpu": 24.02, "tokens/trainable": 145978} +{"epoch": 1.2578125, "grad_norm": 0.6467728614807129, "learning_rate": 4.004940255778431e-05, "loss": 0.014152919873595238, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.01425, "step": 322, "tokens/total": 11210832, "tokens/train_per_sec_per_gpu": 23.67, "tokens/trainable": 146422} +{"epoch": 1.26171875, "grad_norm": 0.06803900003433228, "learning_rate": 3.977591418134619e-05, "loss": 0.0003742177505046129, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.56, "memory/max_allocated (GiB)": 38.56, "ppl": 1.00037, "step": 323, "tokens/total": 11245984, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 146863} +{"epoch": 1.265625, "grad_norm": 0.42967674136161804, "learning_rate": 3.95030593412612e-05, "loss": 0.003911743871867657, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00392, "step": 324, "tokens/total": 11281360, "tokens/train_per_sec_per_gpu": 24.28, "tokens/trainable": 147286} +{"epoch": 1.26953125, "grad_norm": 0.7150908708572388, "learning_rate": 3.923084939213296e-05, "loss": 0.009109622798860073, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.88, "memory/max_allocated (GiB)": 35.88, "ppl": 1.00915, "step": 325, "tokens/total": 11312176, "tokens/train_per_sec_per_gpu": 27.38, "tokens/trainable": 147708} +{"epoch": 1.2734375, "grad_norm": 1.1744827032089233, "learning_rate": 3.895929566172861e-05, "loss": 0.017712850123643875, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.01787, "step": 326, "tokens/total": 11348192, "tokens/train_per_sec_per_gpu": 24.98, "tokens/trainable": 148172} +{"epoch": 1.27734375, "grad_norm": 0.6546609401702881, "learning_rate": 3.868840945050728e-05, "loss": 0.008757010102272034, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.05, "memory/max_allocated (GiB)": 39.05, "ppl": 1.0088, "step": 327, "tokens/total": 11386704, "tokens/train_per_sec_per_gpu": 21.38, "tokens/trainable": 148604} +{"epoch": 1.28125, "grad_norm": 0.14320208132266998, "learning_rate": 3.841820203114995e-05, "loss": 0.0006735241040587425, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00067, "step": 328, "tokens/total": 11418448, "tokens/train_per_sec_per_gpu": 28.83, "tokens/trainable": 149051} +{"epoch": 1.28515625, "grad_norm": 2.4158108234405518, "learning_rate": 3.814868464809027e-05, "loss": 0.0027324878610670567, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.68, "memory/max_allocated (GiB)": 36.68, "ppl": 1.00274, "step": 329, "tokens/total": 11450304, "tokens/train_per_sec_per_gpu": 25.63, "tokens/trainable": 149453} +{"epoch": 1.2890625, "grad_norm": 0.3656834065914154, "learning_rate": 3.787986851704667e-05, "loss": 0.005118336528539658, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.29, "memory/max_allocated (GiB)": 37.29, "ppl": 1.00513, "step": 330, "tokens/total": 11484048, "tokens/train_per_sec_per_gpu": 25.48, "tokens/trainable": 149883} +{"epoch": 1.29296875, "grad_norm": 0.018396778032183647, "learning_rate": 3.7611764824555654e-05, "loss": 0.00012253603199496865, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00012, "step": 331, "tokens/total": 11516032, "tokens/train_per_sec_per_gpu": 30.69, "tokens/trainable": 150357} +{"epoch": 1.296875, "grad_norm": 0.5686500072479248, "learning_rate": 3.734438472750619e-05, "loss": 0.017064878717064857, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.4, "memory/max_allocated (GiB)": 39.4, "ppl": 1.01721, "step": 332, "tokens/total": 11555968, "tokens/train_per_sec_per_gpu": 22.96, "tokens/trainable": 150828} +{"epoch": 1.30078125, "grad_norm": 0.13482479751110077, "learning_rate": 3.707773935267552e-05, "loss": 0.0006157750030979514, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.46, "memory/max_allocated (GiB)": 38.46, "ppl": 1.00062, "step": 333, "tokens/total": 11592880, "tokens/train_per_sec_per_gpu": 26.69, "tokens/trainable": 151314} +{"epoch": 1.3046875, "grad_norm": 0.10960053652524948, "learning_rate": 3.68118397962661e-05, "loss": 0.0010772219393402338, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.5, "memory/max_allocated (GiB)": 38.5, "ppl": 1.00108, "step": 334, "tokens/total": 11627792, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 151722} +{"epoch": 1.30859375, "grad_norm": 0.1267562210559845, "learning_rate": 3.654669712344384e-05, "loss": 0.001150321215391159, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00115, "step": 335, "tokens/total": 11658832, "tokens/train_per_sec_per_gpu": 31.35, "tokens/trainable": 152178} +{"epoch": 1.3125, "grad_norm": 0.004735299851745367, "learning_rate": 3.628232236787763e-05, "loss": 5.44372305739671e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.96, "memory/max_allocated (GiB)": 37.96, "ppl": 1.00005, "step": 336, "tokens/total": 11694464, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 152617} +{"epoch": 1.31640625, "grad_norm": 0.129337877035141, "learning_rate": 3.6018726531280144e-05, "loss": 0.0012166800443083048, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00122, "step": 337, "tokens/total": 11732176, "tokens/train_per_sec_per_gpu": 28.08, "tokens/trainable": 153108} +{"epoch": 1.3203125, "grad_norm": 0.15325579047203064, "learning_rate": 3.575592058295017e-05, "loss": 0.0011003719409927726, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.14, "memory/max_allocated (GiB)": 37.14, "ppl": 1.0011, "step": 338, "tokens/total": 11766032, "tokens/train_per_sec_per_gpu": 28.34, "tokens/trainable": 153589} +{"epoch": 1.32421875, "grad_norm": 0.2811241149902344, "learning_rate": 3.549391545931585e-05, "loss": 0.0023588885087519884, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00236, "step": 339, "tokens/total": 11800640, "tokens/train_per_sec_per_gpu": 27.49, "tokens/trainable": 154049} +{"epoch": 1.328125, "grad_norm": 0.10832665115594864, "learning_rate": 3.5232722063479914e-05, "loss": 0.0004960701335221529, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.0005, "step": 340, "tokens/total": 11834352, "tokens/train_per_sec_per_gpu": 23.96, "tokens/trainable": 154498} +{"epoch": 1.33203125, "grad_norm": 0.2795685827732086, "learning_rate": 3.49723512647657e-05, "loss": 0.0008559411508031189, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00086, "step": 341, "tokens/total": 11873056, "tokens/train_per_sec_per_gpu": 24.6, "tokens/trainable": 154998} +{"epoch": 1.3359375, "grad_norm": 0.6090726256370544, "learning_rate": 3.471281389826491e-05, "loss": 0.007887986488640308, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.48, "memory/max_allocated (GiB)": 36.48, "ppl": 1.00792, "step": 342, "tokens/total": 11904192, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 155403} +{"epoch": 1.33984375, "grad_norm": 0.46598631143569946, "learning_rate": 3.4454120764386764e-05, "loss": 0.005299981217831373, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.66, "memory/max_allocated (GiB)": 38.66, "ppl": 1.00531, "step": 343, "tokens/total": 11942384, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 155858} +{"epoch": 1.34375, "grad_norm": 0.2032434195280075, "learning_rate": 3.4196282628408526e-05, "loss": 0.0023453652393072844, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.66, "memory/max_allocated (GiB)": 37.66, "ppl": 1.00235, "step": 344, "tokens/total": 11976624, "tokens/train_per_sec_per_gpu": 25.07, "tokens/trainable": 156303} +{"epoch": 1.34765625, "grad_norm": 0.030438942834734917, "learning_rate": 3.3939310220027456e-05, "loss": 0.0002629078517202288, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.68, "memory/max_allocated (GiB)": 37.68, "ppl": 1.00026, "step": 345, "tokens/total": 12010768, "tokens/train_per_sec_per_gpu": 28.56, "tokens/trainable": 156798} +{"epoch": 1.3515625, "grad_norm": 0.024013642221689224, "learning_rate": 3.3683214232914404e-05, "loss": 0.00023525326105300337, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.61, "memory/max_allocated (GiB)": 38.61, "ppl": 1.00024, "step": 346, "tokens/total": 12046224, "tokens/train_per_sec_per_gpu": 25.95, "tokens/trainable": 157271} +{"epoch": 1.35546875, "grad_norm": 0.008332701399922371, "learning_rate": 3.342800532426873e-05, "loss": 9.094169945456088e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.73, "memory/max_allocated (GiB)": 38.73, "ppl": 1.00009, "step": 347, "tokens/total": 12082160, "tokens/train_per_sec_per_gpu": 25.25, "tokens/trainable": 157737} +{"epoch": 1.359375, "grad_norm": 0.40410134196281433, "learning_rate": 3.317369411437484e-05, "loss": 0.0034661816898733377, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00347, "step": 348, "tokens/total": 12118128, "tokens/train_per_sec_per_gpu": 25.98, "tokens/trainable": 158205} +{"epoch": 1.36328125, "grad_norm": 1.6874793767929077, "learning_rate": 3.292029118616024e-05, "loss": 0.0018556425347924232, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.08, "memory/max_allocated (GiB)": 37.08, "ppl": 1.00186, "step": 349, "tokens/total": 12151264, "tokens/train_per_sec_per_gpu": 25.4, "tokens/trainable": 158628} +{"epoch": 1.3671875, "grad_norm": 0.5043838024139404, "learning_rate": 3.266780708475511e-05, "loss": 0.005803273059427738, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00582, "step": 350, "tokens/total": 12186800, "tokens/train_per_sec_per_gpu": 24.86, "tokens/trainable": 159106} +{"epoch": 1.37109375, "grad_norm": 0.3341294229030609, "learning_rate": 3.241625231705354e-05, "loss": 0.00257191713899374, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.57, "memory/max_allocated (GiB)": 38.57, "ppl": 1.00258, "step": 351, "tokens/total": 12222944, "tokens/train_per_sec_per_gpu": 24.82, "tokens/trainable": 159565} +{"epoch": 1.375, "grad_norm": 0.020570218563079834, "learning_rate": 3.216563735127618e-05, "loss": 0.00025106826797127724, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.15, "memory/max_allocated (GiB)": 39.15, "ppl": 1.00025, "step": 352, "tokens/total": 12258208, "tokens/train_per_sec_per_gpu": 25.36, "tokens/trainable": 160042} +{"epoch": 1.37890625, "grad_norm": 0.45583200454711914, "learning_rate": 3.191597261653475e-05, "loss": 0.006734559312462807, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00676, "step": 353, "tokens/total": 12295024, "tokens/train_per_sec_per_gpu": 23.55, "tokens/trainable": 160502} +{"epoch": 1.3828125, "grad_norm": 0.5891334414482117, "learning_rate": 3.166726850239794e-05, "loss": 0.008938287384808064, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.97, "memory/max_allocated (GiB)": 36.97, "ppl": 1.00898, "step": 354, "tokens/total": 12328064, "tokens/train_per_sec_per_gpu": 27.24, "tokens/trainable": 160950} +{"epoch": 1.38671875, "grad_norm": 0.004760067444294691, "learning_rate": 3.141953535845912e-05, "loss": 4.59898219560273e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.34, "memory/max_allocated (GiB)": 36.34, "ppl": 1.00005, "step": 355, "tokens/total": 12359504, "tokens/train_per_sec_per_gpu": 30.14, "tokens/trainable": 161400} +{"epoch": 1.390625, "grad_norm": 0.3725301921367645, "learning_rate": 3.11727834939056e-05, "loss": 0.002858710242435336, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00286, "step": 356, "tokens/total": 12391824, "tokens/train_per_sec_per_gpu": 25.17, "tokens/trainable": 161863} +{"epoch": 1.39453125, "grad_norm": 0.3290911614894867, "learning_rate": 3.092702317708967e-05, "loss": 0.008449516259133816, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.84, "memory/max_allocated (GiB)": 36.84, "ppl": 1.00849, "step": 357, "tokens/total": 12425408, "tokens/train_per_sec_per_gpu": 28.66, "tokens/trainable": 162319} +{"epoch": 1.3984375, "grad_norm": 0.303341805934906, "learning_rate": 3.0682264635101276e-05, "loss": 0.0017248715739697218, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.49, "memory/max_allocated (GiB)": 37.49, "ppl": 1.00173, "step": 358, "tokens/total": 12458160, "tokens/train_per_sec_per_gpu": 28.06, "tokens/trainable": 162771} +{"epoch": 1.40234375, "grad_norm": 0.00862891599535942, "learning_rate": 3.0438518053342407e-05, "loss": 0.00013731466606259346, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00014, "step": 359, "tokens/total": 12497104, "tokens/train_per_sec_per_gpu": 22.08, "tokens/trainable": 163243} +{"epoch": 1.40625, "grad_norm": 0.5278812646865845, "learning_rate": 3.0195793575103266e-05, "loss": 0.005015386268496513, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00503, "step": 360, "tokens/total": 12536064, "tokens/train_per_sec_per_gpu": 21.11, "tokens/trainable": 163707} +{"epoch": 1.41015625, "grad_norm": 0.141533762216568, "learning_rate": 2.9954101301140146e-05, "loss": 0.0013897041790187359, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.57, "memory/max_allocated (GiB)": 37.57, "ppl": 1.00139, "step": 361, "tokens/total": 12571488, "tokens/train_per_sec_per_gpu": 24.4, "tokens/trainable": 164157} +{"epoch": 1.4140625, "grad_norm": 0.2104320079088211, "learning_rate": 2.9713451289255123e-05, "loss": 0.0009856335818767548, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.63, "memory/max_allocated (GiB)": 36.63, "ppl": 1.00099, "step": 362, "tokens/total": 12600864, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 164598} +{"epoch": 1.41796875, "grad_norm": 0.2422262728214264, "learning_rate": 2.9473853553877484e-05, "loss": 0.002323160646483302, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.42, "memory/max_allocated (GiB)": 38.42, "ppl": 1.00233, "step": 363, "tokens/total": 12635024, "tokens/train_per_sec_per_gpu": 27.03, "tokens/trainable": 165072} +{"epoch": 1.421875, "grad_norm": 0.19793476164340973, "learning_rate": 2.9235318065647e-05, "loss": 0.0023486418649554253, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.42, "memory/max_allocated (GiB)": 39.42, "ppl": 1.00235, "step": 364, "tokens/total": 12674736, "tokens/train_per_sec_per_gpu": 24.75, "tokens/trainable": 165533} +{"epoch": 1.42578125, "grad_norm": 0.017844239249825478, "learning_rate": 2.8997854750998964e-05, "loss": 0.0001600450777914375, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.46, "memory/max_allocated (GiB)": 39.46, "ppl": 1.00016, "step": 365, "tokens/total": 12714256, "tokens/train_per_sec_per_gpu": 23.0, "tokens/trainable": 165998} +{"epoch": 1.4296875, "grad_norm": 0.02212539128959179, "learning_rate": 2.8761473491751258e-05, "loss": 0.00017600657884031534, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.75, "memory/max_allocated (GiB)": 36.75, "ppl": 1.00018, "step": 366, "tokens/total": 12747504, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 166433} +{"epoch": 1.43359375, "grad_norm": 0.427540123462677, "learning_rate": 2.8526184124692883e-05, "loss": 0.00826314277946949, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.0083, "step": 367, "tokens/total": 12783872, "tokens/train_per_sec_per_gpu": 20.25, "tokens/trainable": 166871} +{"epoch": 1.4375, "grad_norm": 0.5444613695144653, "learning_rate": 2.829199644117484e-05, "loss": 0.011646736413240433, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.85, "memory/max_allocated (GiB)": 37.85, "ppl": 1.01171, "step": 368, "tokens/total": 12817856, "tokens/train_per_sec_per_gpu": 24.14, "tokens/trainable": 167281} +{"epoch": 1.44140625, "grad_norm": 0.29842621088027954, "learning_rate": 2.8058920186702553e-05, "loss": 0.005181244108825922, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00519, "step": 369, "tokens/total": 12855472, "tokens/train_per_sec_per_gpu": 24.8, "tokens/trainable": 167754} +{"epoch": 1.4453125, "grad_norm": 0.0051379017531871796, "learning_rate": 2.782696506053033e-05, "loss": 9.246310219168663e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00009, "step": 370, "tokens/total": 12893344, "tokens/train_per_sec_per_gpu": 22.19, "tokens/trainable": 168186} +{"epoch": 1.44921875, "grad_norm": 0.28771868348121643, "learning_rate": 2.7596140715257824e-05, "loss": 0.002678230870515108, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.74, "memory/max_allocated (GiB)": 38.74, "ppl": 1.00268, "step": 371, "tokens/total": 12928512, "tokens/train_per_sec_per_gpu": 25.54, "tokens/trainable": 168651} +{"epoch": 1.453125, "grad_norm": 0.04575556144118309, "learning_rate": 2.7366456756428184e-05, "loss": 0.0002823758695740253, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.46, "memory/max_allocated (GiB)": 37.46, "ppl": 1.00028, "step": 372, "tokens/total": 12963120, "tokens/train_per_sec_per_gpu": 26.98, "tokens/trainable": 169108} +{"epoch": 1.45703125, "grad_norm": 0.2429930567741394, "learning_rate": 2.7137922742128486e-05, "loss": 0.0012873129453510046, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.92, "memory/max_allocated (GiB)": 37.92, "ppl": 1.00129, "step": 373, "tokens/total": 12997648, "tokens/train_per_sec_per_gpu": 26.83, "tokens/trainable": 169607} +{"epoch": 1.4609375, "grad_norm": 0.4451111853122711, "learning_rate": 2.691054818259188e-05, "loss": 0.007133429404348135, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.00716, "step": 374, "tokens/total": 13032496, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 170080} +{"epoch": 1.46484375, "grad_norm": 0.0878291130065918, "learning_rate": 2.6684342539801933e-05, "loss": 0.0010703718289732933, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.56, "memory/max_allocated (GiB)": 36.56, "ppl": 1.00107, "step": 375, "tokens/total": 13064016, "tokens/train_per_sec_per_gpu": 28.61, "tokens/trainable": 170532} +{"epoch": 1.46875, "grad_norm": 0.2856180965900421, "learning_rate": 2.645931522709877e-05, "loss": 0.0013454864965751767, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.2, "memory/max_allocated (GiB)": 39.2, "ppl": 1.00135, "step": 376, "tokens/total": 13099408, "tokens/train_per_sec_per_gpu": 28.76, "tokens/trainable": 171012} +{"epoch": 1.47265625, "grad_norm": 0.49029210209846497, "learning_rate": 2.6235475608787365e-05, "loss": 0.002783253090456128, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00279, "step": 377, "tokens/total": 13135088, "tokens/train_per_sec_per_gpu": 25.26, "tokens/trainable": 171473} +{"epoch": 1.4765625, "grad_norm": 0.10827801376581192, "learning_rate": 2.6012832999747916e-05, "loss": 0.0018938868306577206, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.46, "memory/max_allocated (GiB)": 36.46, "ppl": 1.0019, "step": 378, "tokens/total": 13166208, "tokens/train_per_sec_per_gpu": 29.0, "tokens/trainable": 171920} +{"epoch": 1.48046875, "grad_norm": 0.42413467168807983, "learning_rate": 2.579139666504821e-05, "loss": 0.022013423964381218, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.02226, "step": 379, "tokens/total": 13206192, "tokens/train_per_sec_per_gpu": 21.91, "tokens/trainable": 172366} +{"epoch": 1.484375, "grad_norm": 0.16861702501773834, "learning_rate": 2.557117581955798e-05, "loss": 0.0007429906399920583, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00074, "step": 380, "tokens/total": 13240032, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 172832} +{"epoch": 1.48828125, "grad_norm": 0.07656534016132355, "learning_rate": 2.5352179627565532e-05, "loss": 0.0007513999589718878, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00075, "step": 381, "tokens/total": 13274144, "tokens/train_per_sec_per_gpu": 25.41, "tokens/trainable": 173272} +{"epoch": 1.4921875, "grad_norm": 0.6613176465034485, "learning_rate": 2.5134417202396277e-05, "loss": 0.007952416315674782, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.6, "memory/max_allocated (GiB)": 38.6, "ppl": 1.00798, "step": 382, "tokens/total": 13308128, "tokens/train_per_sec_per_gpu": 23.83, "tokens/trainable": 173679} +{"epoch": 1.49609375, "grad_norm": 0.24728795886039734, "learning_rate": 2.491789760603361e-05, "loss": 0.002179680857807398, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.00218, "step": 383, "tokens/total": 13344544, "tokens/train_per_sec_per_gpu": 23.06, "tokens/trainable": 174109} +{"epoch": 1.5, "grad_norm": 0.1289031058549881, "learning_rate": 2.4702629848741764e-05, "loss": 0.001106928219087422, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.6, "memory/max_allocated (GiB)": 37.6, "ppl": 1.00111, "step": 384, "tokens/total": 13378592, "tokens/train_per_sec_per_gpu": 26.32, "tokens/trainable": 174564} +{"epoch": 1.50390625, "grad_norm": 0.5123085379600525, "learning_rate": 2.4488622888690785e-05, "loss": 0.015107223764061928, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.04, "memory/max_allocated (GiB)": 37.04, "ppl": 1.01522, "step": 385, "tokens/total": 13412048, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 175004} +{"epoch": 1.5078125, "grad_norm": 0.030340345576405525, "learning_rate": 2.427588563158384e-05, "loss": 0.00034379996941424906, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.00034, "step": 386, "tokens/total": 13446384, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 175490} +{"epoch": 1.51171875, "grad_norm": 0.11403896659612656, "learning_rate": 2.406442693028651e-05, "loss": 0.001409953460097313, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.63, "memory/max_allocated (GiB)": 38.63, "ppl": 1.00141, "step": 387, "tokens/total": 13478320, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 175911} +{"epoch": 1.515625, "grad_norm": 0.009696883149445057, "learning_rate": 2.3854255584458547e-05, "loss": 0.00016735793906264007, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00017, "step": 388, "tokens/total": 13516912, "tokens/train_per_sec_per_gpu": 21.44, "tokens/trainable": 176338} +{"epoch": 1.51953125, "grad_norm": 0.19666066765785217, "learning_rate": 2.3645380340187508e-05, "loss": 0.00328278262168169, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.98, "memory/max_allocated (GiB)": 38.98, "ppl": 1.00329, "step": 389, "tokens/total": 13552592, "tokens/train_per_sec_per_gpu": 25.92, "tokens/trainable": 176819} +{"epoch": 1.5234375, "grad_norm": 0.5583411455154419, "learning_rate": 2.3437809889624914e-05, "loss": 0.004744517616927624, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.54, "memory/max_allocated (GiB)": 36.54, "ppl": 1.00476, "step": 390, "tokens/total": 13583856, "tokens/train_per_sec_per_gpu": 28.97, "tokens/trainable": 177270} +{"epoch": 1.52734375, "grad_norm": 0.10284982621669769, "learning_rate": 2.3231552870624487e-05, "loss": 0.0006621374632231891, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.47, "memory/max_allocated (GiB)": 38.47, "ppl": 1.00066, "step": 391, "tokens/total": 13614864, "tokens/train_per_sec_per_gpu": 28.94, "tokens/trainable": 177696} +{"epoch": 1.53125, "grad_norm": 0.2074676901102066, "learning_rate": 2.3026617866382657e-05, "loss": 0.0017339906189590693, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.53, "memory/max_allocated (GiB)": 36.53, "ppl": 1.00174, "step": 392, "tokens/total": 13647616, "tokens/train_per_sec_per_gpu": 29.66, "tokens/trainable": 178193} +{"epoch": 1.53515625, "grad_norm": 0.07604361325502396, "learning_rate": 2.2823013405081507e-05, "loss": 0.0008006435818970203, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.34, "memory/max_allocated (GiB)": 39.34, "ppl": 1.0008, "step": 393, "tokens/total": 13685696, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 178690} +{"epoch": 1.5390625, "grad_norm": 0.04959225654602051, "learning_rate": 2.2620747959533722e-05, "loss": 0.00038986679282970726, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.76, "memory/max_allocated (GiB)": 36.76, "ppl": 1.00039, "step": 394, "tokens/total": 13718768, "tokens/train_per_sec_per_gpu": 30.49, "tokens/trainable": 179187} +{"epoch": 1.54296875, "grad_norm": 0.04030602425336838, "learning_rate": 2.2419829946830123e-05, "loss": 0.0005295201553963125, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00053, "step": 395, "tokens/total": 13754960, "tokens/train_per_sec_per_gpu": 24.06, "tokens/trainable": 179638} +{"epoch": 1.546875, "grad_norm": 0.16147907078266144, "learning_rate": 2.2220267727989325e-05, "loss": 0.0022257536184042692, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00223, "step": 396, "tokens/total": 13789104, "tokens/train_per_sec_per_gpu": 25.51, "tokens/trainable": 180091} +{"epoch": 1.55078125, "grad_norm": 0.13644173741340637, "learning_rate": 2.202206960760984e-05, "loss": 0.001973393838852644, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.89, "memory/max_allocated (GiB)": 38.89, "ppl": 1.00198, "step": 397, "tokens/total": 13827248, "tokens/train_per_sec_per_gpu": 24.53, "tokens/trainable": 180598} +{"epoch": 1.5546875, "grad_norm": 0.17517144978046417, "learning_rate": 2.182524383352446e-05, "loss": 0.0029745926149189472, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00298, "step": 398, "tokens/total": 13863408, "tokens/train_per_sec_per_gpu": 23.81, "tokens/trainable": 181051} +{"epoch": 1.55859375, "grad_norm": 0.39292803406715393, "learning_rate": 2.1629798596457056e-05, "loss": 0.01645696349442005, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.52, "memory/max_allocated (GiB)": 36.52, "ppl": 1.01659, "step": 399, "tokens/total": 13895728, "tokens/train_per_sec_per_gpu": 26.37, "tokens/trainable": 181501} +{"epoch": 1.5625, "grad_norm": 0.025967111811041832, "learning_rate": 2.1435742029681725e-05, "loss": 0.0004191006300970912, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.08, "memory/max_allocated (GiB)": 36.08, "ppl": 1.00042, "step": 400, "tokens/total": 13926048, "tokens/train_per_sec_per_gpu": 28.0, "tokens/trainable": 181918} +{"epoch": 1.56640625, "grad_norm": 0.04479897767305374, "learning_rate": 2.124308220868431e-05, "loss": 0.00026676285779103637, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.67, "memory/max_allocated (GiB)": 38.67, "ppl": 1.00027, "step": 401, "tokens/total": 13961440, "tokens/train_per_sec_per_gpu": 24.1, "tokens/trainable": 182373} +{"epoch": 1.5703125, "grad_norm": 0.3229577839374542, "learning_rate": 2.105182715082638e-05, "loss": 0.0020829092245548964, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.89, "memory/max_allocated (GiB)": 36.89, "ppl": 1.00209, "step": 402, "tokens/total": 13995040, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 182850} +{"epoch": 1.57421875, "grad_norm": 0.04477771744132042, "learning_rate": 2.0861984815011552e-05, "loss": 0.0003504430642351508, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.16, "memory/max_allocated (GiB)": 38.16, "ppl": 1.00035, "step": 403, "tokens/total": 14030560, "tokens/train_per_sec_per_gpu": 22.42, "tokens/trainable": 183281} +{"epoch": 1.578125, "grad_norm": 0.21391765773296356, "learning_rate": 2.0673563101354323e-05, "loss": 0.0010880237678065896, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00109, "step": 404, "tokens/total": 14065840, "tokens/train_per_sec_per_gpu": 22.48, "tokens/trainable": 183711} +{"epoch": 1.58203125, "grad_norm": 0.15335458517074585, "learning_rate": 2.0486569850851317e-05, "loss": 0.0017563470173627138, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.25, "memory/max_allocated (GiB)": 38.25, "ppl": 1.00176, "step": 405, "tokens/total": 14100848, "tokens/train_per_sec_per_gpu": 25.67, "tokens/trainable": 184174} +{"epoch": 1.5859375, "grad_norm": 0.020001186057925224, "learning_rate": 2.0301012845054956e-05, "loss": 0.00032516694045625627, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.03, "memory/max_allocated (GiB)": 39.03, "ppl": 1.00033, "step": 406, "tokens/total": 14139600, "tokens/train_per_sec_per_gpu": 20.46, "tokens/trainable": 184589} +{"epoch": 1.58984375, "grad_norm": 0.11474430561065674, "learning_rate": 2.011689980574966e-05, "loss": 0.001670363824814558, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.06, "memory/max_allocated (GiB)": 38.06, "ppl": 1.00167, "step": 407, "tokens/total": 14175904, "tokens/train_per_sec_per_gpu": 22.68, "tokens/trainable": 185009} +{"epoch": 1.59375, "grad_norm": 0.07059294730424881, "learning_rate": 1.993423839463052e-05, "loss": 0.00034554023295640945, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.09, "memory/max_allocated (GiB)": 38.09, "ppl": 1.00035, "step": 408, "tokens/total": 14209168, "tokens/train_per_sec_per_gpu": 31.61, "tokens/trainable": 185466} +{"epoch": 1.59765625, "grad_norm": 0.18652531504631042, "learning_rate": 1.975303621298445e-05, "loss": 0.00020865671103820205, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.24, "memory/max_allocated (GiB)": 39.24, "ppl": 1.00021, "step": 409, "tokens/total": 14246208, "tokens/train_per_sec_per_gpu": 25.22, "tokens/trainable": 185918} +{"epoch": 1.6015625, "grad_norm": 0.740872859954834, "learning_rate": 1.957330080137385e-05, "loss": 0.001399340806528926, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.31, "memory/max_allocated (GiB)": 36.31, "ppl": 1.0014, "step": 410, "tokens/total": 14277072, "tokens/train_per_sec_per_gpu": 28.79, "tokens/trainable": 186369} +{"epoch": 1.60546875, "grad_norm": 0.11544816195964813, "learning_rate": 1.9395039639322864e-05, "loss": 0.0011494482168927789, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.77, "memory/max_allocated (GiB)": 38.77, "ppl": 1.00115, "step": 411, "tokens/total": 14313120, "tokens/train_per_sec_per_gpu": 26.11, "tokens/trainable": 186844} +{"epoch": 1.609375, "grad_norm": 0.01287645474076271, "learning_rate": 1.9218260145006073e-05, "loss": 0.00010904463124461472, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.52, "memory/max_allocated (GiB)": 37.52, "ppl": 1.00011, "step": 412, "tokens/total": 14348368, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 187330} +{"epoch": 1.61328125, "grad_norm": 0.17310501635074615, "learning_rate": 1.904296967493982e-05, "loss": 0.0021268907003104687, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 36.91, "memory/max_allocated (GiB)": 36.91, "ppl": 1.00213, "step": 413, "tokens/total": 14380528, "tokens/train_per_sec_per_gpu": 29.37, "tokens/trainable": 187786} +{"epoch": 1.6171875, "grad_norm": 0.16469748318195343, "learning_rate": 1.8869175523676064e-05, "loss": 0.0023324957583099604, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00234, "step": 414, "tokens/total": 14417216, "tokens/train_per_sec_per_gpu": 26.36, "tokens/trainable": 188281} +{"epoch": 1.62109375, "grad_norm": 0.018500683829188347, "learning_rate": 1.869688492349885e-05, "loss": 0.0001746786874718964, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.55, "memory/max_allocated (GiB)": 37.55, "ppl": 1.00017, "step": 415, "tokens/total": 14450672, "tokens/train_per_sec_per_gpu": 27.12, "tokens/trainable": 188733} +{"epoch": 1.625, "grad_norm": 0.09585463255643845, "learning_rate": 1.85261050441233e-05, "loss": 0.0009471923112869263, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.78, "memory/max_allocated (GiB)": 37.78, "ppl": 1.00095, "step": 416, "tokens/total": 14486464, "tokens/train_per_sec_per_gpu": 24.18, "tokens/trainable": 189189} +{"epoch": 1.62890625, "grad_norm": 0.9904699921607971, "learning_rate": 1.8356842992397304e-05, "loss": 0.009134965017437935, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.13, "memory/max_allocated (GiB)": 37.13, "ppl": 1.00918, "step": 417, "tokens/total": 14519984, "tokens/train_per_sec_per_gpu": 26.79, "tokens/trainable": 189629} +{"epoch": 1.6328125, "grad_norm": 0.07084454596042633, "learning_rate": 1.8189105812005714e-05, "loss": 0.0008301247144117951, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.0, "memory/max_allocated (GiB)": 39.0, "ppl": 1.00083, "step": 418, "tokens/total": 14556800, "tokens/train_per_sec_per_gpu": 22.05, "tokens/trainable": 190039} +{"epoch": 1.63671875, "grad_norm": 0.2778157889842987, "learning_rate": 1.802290048317732e-05, "loss": 0.003368410049006343, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00337, "step": 419, "tokens/total": 14590480, "tokens/train_per_sec_per_gpu": 23.23, "tokens/trainable": 190457} +{"epoch": 1.640625, "grad_norm": 0.016981925815343857, "learning_rate": 1.785823392239424e-05, "loss": 0.00012425988097675145, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 37.9, "memory/max_allocated (GiB)": 37.9, "ppl": 1.00012, "step": 420, "tokens/total": 14624064, "tokens/train_per_sec_per_gpu": 25.97, "tokens/trainable": 190898} +{"epoch": 1.64453125, "grad_norm": 0.4231007695198059, "learning_rate": 1.7695112982104225e-05, "loss": 0.005061562173068523, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00507, "step": 421, "tokens/total": 14659072, "tokens/train_per_sec_per_gpu": 24.67, "tokens/trainable": 191337} +{"epoch": 1.6484375, "grad_norm": 0.2275489717721939, "learning_rate": 1.7533544450435433e-05, "loss": 0.0022434245329350233, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.79, "memory/max_allocated (GiB)": 35.79, "ppl": 1.00225, "step": 422, "tokens/total": 14688624, "tokens/train_per_sec_per_gpu": 29.82, "tokens/trainable": 191770} +{"epoch": 1.65234375, "grad_norm": 0.006618031766265631, "learning_rate": 1.7373535050913946e-05, "loss": 5.485107976710424e-05, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.83, "memory/max_allocated (GiB)": 38.83, "ppl": 1.00005, "step": 423, "tokens/total": 14725024, "tokens/train_per_sec_per_gpu": 21.74, "tokens/trainable": 192207} +{"epoch": 1.65625, "grad_norm": 0.16173480451107025, "learning_rate": 1.721509144218405e-05, "loss": 0.0020012944005429745, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 35.85, "memory/max_allocated (GiB)": 35.85, "ppl": 1.002, "step": 424, "tokens/total": 14755344, "tokens/train_per_sec_per_gpu": 29.62, "tokens/trainable": 192637} +{"epoch": 1.66015625, "grad_norm": 0.012736567296087742, "learning_rate": 1.705822021773101e-05, "loss": 0.00019328840426169336, "memory/device_reserved (GiB)": 42.34, "memory/max_active (GiB)": 38.64, "memory/max_allocated (GiB)": 38.64, "ppl": 1.00019, "step": 425, "tokens/total": 14789776, "tokens/train_per_sec_per_gpu": 27.8, "tokens/trainable": 193123} +{"epoch": 1.6640625, "grad_norm": 0.3201669454574585, "learning_rate": 1.69029279056068e-05, "loss": 0.015019687823951244, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.94, "memory/max_allocated (GiB)": 38.94, "ppl": 1.01513, "step": 426, "tokens/total": 14826896, "tokens/train_per_sec_per_gpu": 24.94, "tokens/trainable": 193564} +{"epoch": 1.66796875, "grad_norm": 0.30655333399772644, "learning_rate": 1.6749220968158415e-05, "loss": 0.004036834929138422, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.38, "memory/max_allocated (GiB)": 39.38, "ppl": 1.00404, "step": 427, "tokens/total": 14862192, "tokens/train_per_sec_per_gpu": 28.88, "tokens/trainable": 194070} +{"epoch": 1.671875, "grad_norm": 0.1474546194076538, "learning_rate": 1.659710580175893e-05, "loss": 0.00153299979865551, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 35.91, "memory/max_allocated (GiB)": 35.91, "ppl": 1.00153, "step": 428, "tokens/total": 14893264, "tokens/train_per_sec_per_gpu": 31.94, "tokens/trainable": 194569} +{"epoch": 1.67578125, "grad_norm": 0.0293866079300642, "learning_rate": 1.644658873654133e-05, "loss": 0.00040839705616235733, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.03, "memory/max_allocated (GiB)": 37.03, "ppl": 1.00041, "step": 429, "tokens/total": 14925712, "tokens/train_per_sec_per_gpu": 30.22, "tokens/trainable": 195006} +{"epoch": 1.6796875, "grad_norm": 0.12265431880950928, "learning_rate": 1.629767603613508e-05, "loss": 0.001053130836226046, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.24, "memory/max_allocated (GiB)": 36.24, "ppl": 1.00105, "step": 430, "tokens/total": 14956944, "tokens/train_per_sec_per_gpu": 25.57, "tokens/trainable": 195406} +{"epoch": 1.68359375, "grad_norm": 0.37998107075691223, "learning_rate": 1.615037389740547e-05, "loss": 0.004547814838588238, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.36, "memory/max_allocated (GiB)": 37.36, "ppl": 1.00456, "step": 431, "tokens/total": 14991488, "tokens/train_per_sec_per_gpu": 22.8, "tokens/trainable": 195850} +{"epoch": 1.6875, "grad_norm": 0.36520135402679443, "learning_rate": 1.600468845019576e-05, "loss": 0.007853361777961254, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00788, "step": 432, "tokens/total": 15029856, "tokens/train_per_sec_per_gpu": 23.71, "tokens/trainable": 196330} +{"epoch": 1.69140625, "grad_norm": 0.011735337786376476, "learning_rate": 1.5860625757072092e-05, "loss": 9.173118451144546e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.38, "memory/max_allocated (GiB)": 38.38, "ppl": 1.00009, "step": 433, "tokens/total": 15064864, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 196778} +{"epoch": 1.6953125, "grad_norm": 0.012387464754283428, "learning_rate": 1.571819181307116e-05, "loss": 0.0001442567736376077, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.2, "memory/max_allocated (GiB)": 38.2, "ppl": 1.00014, "step": 434, "tokens/total": 15101440, "tokens/train_per_sec_per_gpu": 25.53, "tokens/trainable": 197250} +{"epoch": 1.69921875, "grad_norm": 1.2071713209152222, "learning_rate": 1.557739254545075e-05, "loss": 0.0023768141400069, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.28, "memory/max_allocated (GiB)": 36.28, "ppl": 1.00238, "step": 435, "tokens/total": 15132464, "tokens/train_per_sec_per_gpu": 32.49, "tokens/trainable": 197722} +{"epoch": 1.703125, "grad_norm": 0.15526556968688965, "learning_rate": 1.543823381344311e-05, "loss": 0.0021712318994104862, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.75, "memory/max_allocated (GiB)": 37.75, "ppl": 1.00217, "step": 436, "tokens/total": 15164960, "tokens/train_per_sec_per_gpu": 30.89, "tokens/trainable": 198196} +{"epoch": 1.70703125, "grad_norm": 0.25397664308547974, "learning_rate": 1.5300721408011114e-05, "loss": 0.002572752069681883, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.08, "memory/max_allocated (GiB)": 39.08, "ppl": 1.00258, "step": 437, "tokens/total": 15200512, "tokens/train_per_sec_per_gpu": 27.55, "tokens/trainable": 198653} +{"epoch": 1.7109375, "grad_norm": 0.22091025114059448, "learning_rate": 1.5164861051607254e-05, "loss": 0.007015953306108713, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 36.9, "memory/max_allocated (GiB)": 36.9, "ppl": 1.00704, "step": 438, "tokens/total": 15233536, "tokens/train_per_sec_per_gpu": 23.38, "tokens/trainable": 199061} +{"epoch": 1.71484375, "grad_norm": 0.009200085885822773, "learning_rate": 1.5030658397935521e-05, "loss": 8.467092993669212e-05, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 38.34, "memory/max_allocated (GiB)": 38.34, "ppl": 1.00008, "step": 439, "tokens/total": 15268896, "tokens/train_per_sec_per_gpu": 25.34, "tokens/trainable": 199530} +{"epoch": 1.71875, "grad_norm": 0.013678290881216526, "learning_rate": 1.4898119031716104e-05, "loss": 0.0001940571964951232, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 37.89, "memory/max_allocated (GiB)": 37.89, "ppl": 1.00019, "step": 440, "tokens/total": 15302288, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 199988} +{"epoch": 1.72265625, "grad_norm": 0.11338362842798233, "learning_rate": 1.476724846845306e-05, "loss": 0.0004946681438013911, "memory/device_reserved (GiB)": 42.36, "memory/max_active (GiB)": 39.21, "memory/max_allocated (GiB)": 39.21, "ppl": 1.00049, "step": 441, "tokens/total": 15340416, "tokens/train_per_sec_per_gpu": 27.18, "tokens/trainable": 200465} +{"epoch": 1.7265625, "grad_norm": 0.018784180283546448, "learning_rate": 1.463805215420471e-05, "loss": 0.0001832921407185495, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.26, "memory/max_allocated (GiB)": 39.26, "ppl": 1.00018, "step": 442, "tokens/total": 15375680, "tokens/train_per_sec_per_gpu": 26.23, "tokens/trainable": 200935} +{"epoch": 1.73046875, "grad_norm": 0.22691427171230316, "learning_rate": 1.451053546535705e-05, "loss": 0.0038385973311960697, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.07, "memory/max_allocated (GiB)": 39.07, "ppl": 1.00385, "step": 443, "tokens/total": 15412560, "tokens/train_per_sec_per_gpu": 24.91, "tokens/trainable": 201391} +{"epoch": 1.734375, "grad_norm": 0.04517659172415733, "learning_rate": 1.438470370840001e-05, "loss": 0.00042660595499910414, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00043, "step": 444, "tokens/total": 15448448, "tokens/train_per_sec_per_gpu": 26.71, "tokens/trainable": 201863} +{"epoch": 1.73828125, "grad_norm": 0.02341851405799389, "learning_rate": 1.4260562119706606e-05, "loss": 0.00041486890404485166, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.00041, "step": 445, "tokens/total": 15481024, "tokens/train_per_sec_per_gpu": 30.02, "tokens/trainable": 202337} +{"epoch": 1.7421875, "grad_norm": 0.4737454950809479, "learning_rate": 1.413811586531508e-05, "loss": 0.005478535778820515, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.76, "memory/max_allocated (GiB)": 37.76, "ppl": 1.00549, "step": 446, "tokens/total": 15514064, "tokens/train_per_sec_per_gpu": 28.5, "tokens/trainable": 202778} +{"epoch": 1.74609375, "grad_norm": 0.05071468651294708, "learning_rate": 1.4017370040713884e-05, "loss": 0.000553867663256824, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.4, "memory/max_allocated (GiB)": 38.4, "ppl": 1.00055, "step": 447, "tokens/total": 15551200, "tokens/train_per_sec_per_gpu": 23.44, "tokens/trainable": 203227} +{"epoch": 1.75, "grad_norm": 0.5461382865905762, "learning_rate": 1.3898329670629645e-05, "loss": 0.009302739053964615, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00935, "step": 448, "tokens/total": 15591040, "tokens/train_per_sec_per_gpu": 20.88, "tokens/trainable": 203672} +{"epoch": 1.75390625, "grad_norm": 0.15832681953907013, "learning_rate": 1.3780999708818058e-05, "loss": 0.00013432273408398032, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.5, "memory/max_allocated (GiB)": 36.5, "ppl": 1.00013, "step": 449, "tokens/total": 15622864, "tokens/train_per_sec_per_gpu": 28.32, "tokens/trainable": 204122} +{"epoch": 1.7578125, "grad_norm": 0.007672576233744621, "learning_rate": 1.3665385037857758e-05, "loss": 0.0001352672406937927, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.96, "memory/max_allocated (GiB)": 36.96, "ppl": 1.00014, "step": 450, "tokens/total": 15656144, "tokens/train_per_sec_per_gpu": 24.79, "tokens/trainable": 204575} +{"epoch": 1.76171875, "grad_norm": 0.21994201838970184, "learning_rate": 1.3551490468947126e-05, "loss": 0.003265283303335309, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.33, "memory/max_allocated (GiB)": 36.33, "ppl": 1.00327, "step": 451, "tokens/total": 15688000, "tokens/train_per_sec_per_gpu": 27.46, "tokens/trainable": 205021} +{"epoch": 1.765625, "grad_norm": 0.15907731652259827, "learning_rate": 1.3439320741704075e-05, "loss": 0.0023239371366798878, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.00233, "step": 452, "tokens/total": 15723104, "tokens/train_per_sec_per_gpu": 24.96, "tokens/trainable": 205444} +{"epoch": 1.76953125, "grad_norm": 0.05552179738879204, "learning_rate": 1.3328880523968808e-05, "loss": 0.0005927427555434406, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.07, "memory/max_allocated (GiB)": 37.07, "ppl": 1.00059, "step": 453, "tokens/total": 15756752, "tokens/train_per_sec_per_gpu": 26.13, "tokens/trainable": 205907} +{"epoch": 1.7734375, "grad_norm": 0.1849990040063858, "learning_rate": 1.3220174411609587e-05, "loss": 0.0013247790047898889, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00133, "step": 454, "tokens/total": 15790576, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 206346} +{"epoch": 1.77734375, "grad_norm": 0.03096238523721695, "learning_rate": 1.3113206928331471e-05, "loss": 0.0003826374595519155, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00038, "step": 455, "tokens/total": 15819344, "tokens/train_per_sec_per_gpu": 30.56, "tokens/trainable": 206808} +{"epoch": 1.78125, "grad_norm": 0.03661311790347099, "learning_rate": 1.300798252548806e-05, "loss": 0.00040016474667936563, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.7, "memory/max_allocated (GiB)": 37.7, "ppl": 1.0004, "step": 456, "tokens/total": 15853776, "tokens/train_per_sec_per_gpu": 27.26, "tokens/trainable": 207260} +{"epoch": 1.78515625, "grad_norm": 0.0037963860668241978, "learning_rate": 1.2904505581896265e-05, "loss": 8.35847167763859e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.25, "memory/max_allocated (GiB)": 39.25, "ppl": 1.00008, "step": 457, "tokens/total": 15892768, "tokens/train_per_sec_per_gpu": 23.26, "tokens/trainable": 207733} +{"epoch": 1.7890625, "grad_norm": 0.0416388064622879, "learning_rate": 1.2802780403654082e-05, "loss": 0.0008168237400241196, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.8, "memory/max_allocated (GiB)": 37.8, "ppl": 1.00082, "step": 458, "tokens/total": 15926656, "tokens/train_per_sec_per_gpu": 26.52, "tokens/trainable": 208180} +{"epoch": 1.79296875, "grad_norm": 0.13182660937309265, "learning_rate": 1.2702811223961408e-05, "loss": 0.0014313369756564498, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.93, "memory/max_allocated (GiB)": 36.93, "ppl": 1.00143, "step": 459, "tokens/total": 15960144, "tokens/train_per_sec_per_gpu": 26.09, "tokens/trainable": 208641} +{"epoch": 1.796875, "grad_norm": 0.36391785740852356, "learning_rate": 1.2604602202943861e-05, "loss": 0.0037350181955844164, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.05, "memory/max_allocated (GiB)": 37.05, "ppl": 1.00374, "step": 460, "tokens/total": 15991232, "tokens/train_per_sec_per_gpu": 31.09, "tokens/trainable": 209084} +{"epoch": 1.80078125, "grad_norm": 0.9378758072853088, "learning_rate": 1.2508157427479686e-05, "loss": 0.007633653935045004, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.99, "memory/max_allocated (GiB)": 36.99, "ppl": 1.00766, "step": 461, "tokens/total": 16023232, "tokens/train_per_sec_per_gpu": 28.22, "tokens/trainable": 209530} +{"epoch": 1.8046875, "grad_norm": 0.19277232885360718, "learning_rate": 1.2413480911029655e-05, "loss": 0.0012064632028341293, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.74, "memory/max_allocated (GiB)": 35.74, "ppl": 1.00121, "step": 462, "tokens/total": 16053744, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 209969} +{"epoch": 1.80859375, "grad_norm": 0.03583604097366333, "learning_rate": 1.2320576593470082e-05, "loss": 0.0004973490140400827, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.65, "memory/max_allocated (GiB)": 38.65, "ppl": 1.0005, "step": 463, "tokens/total": 16088352, "tokens/train_per_sec_per_gpu": 23.78, "tokens/trainable": 210396} +{"epoch": 1.8125, "grad_norm": 0.09323292225599289, "learning_rate": 1.2229448340928828e-05, "loss": 0.0006771157495677471, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00068, "step": 464, "tokens/total": 16127312, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 210904} +{"epoch": 1.81640625, "grad_norm": 0.01404639333486557, "learning_rate": 1.2140099945624458e-05, "loss": 0.0002039596438407898, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.76, "memory/max_allocated (GiB)": 38.76, "ppl": 1.0002, "step": 465, "tokens/total": 16166032, "tokens/train_per_sec_per_gpu": 22.97, "tokens/trainable": 211362} +{"epoch": 1.8203125, "grad_norm": 0.09658671170473099, "learning_rate": 1.205253512570841e-05, "loss": 0.0005106964381411672, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00051, "step": 466, "tokens/total": 16198912, "tokens/train_per_sec_per_gpu": 26.59, "tokens/trainable": 211810} +{"epoch": 1.82421875, "grad_norm": 0.026514090597629547, "learning_rate": 1.1966757525110255e-05, "loss": 0.0004267814219929278, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 35.55, "memory/max_allocated (GiB)": 35.55, "ppl": 1.00043, "step": 467, "tokens/total": 16228672, "tokens/train_per_sec_per_gpu": 30.66, "tokens/trainable": 212261} +{"epoch": 1.828125, "grad_norm": 0.06400958448648453, "learning_rate": 1.1882770713386095e-05, "loss": 0.0007460680790245533, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.5, "memory/max_allocated (GiB)": 39.5, "ppl": 1.00075, "step": 468, "tokens/total": 16265760, "tokens/train_per_sec_per_gpu": 25.96, "tokens/trainable": 212736} +{"epoch": 1.83203125, "grad_norm": 0.28927892446517944, "learning_rate": 1.180057818556998e-05, "loss": 0.003177258651703596, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.43, "memory/max_allocated (GiB)": 38.43, "ppl": 1.00318, "step": 469, "tokens/total": 16301856, "tokens/train_per_sec_per_gpu": 25.62, "tokens/trainable": 213211} +{"epoch": 1.8359375, "grad_norm": 0.007580700796097517, "learning_rate": 1.1720183362028494e-05, "loss": 0.00016651884652674198, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.55, "memory/max_allocated (GiB)": 38.55, "ppl": 1.00017, "step": 470, "tokens/total": 16338336, "tokens/train_per_sec_per_gpu": 24.78, "tokens/trainable": 213676} +{"epoch": 1.83984375, "grad_norm": 0.5595632195472717, "learning_rate": 1.1641589588318387e-05, "loss": 0.0025138044729828835, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00252, "step": 471, "tokens/total": 16375840, "tokens/train_per_sec_per_gpu": 23.22, "tokens/trainable": 214121} +{"epoch": 1.84375, "grad_norm": 0.038976289331912994, "learning_rate": 1.1564800135047418e-05, "loss": 0.00015902549785096198, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.1, "memory/max_allocated (GiB)": 37.1, "ppl": 1.00016, "step": 472, "tokens/total": 16409504, "tokens/train_per_sec_per_gpu": 27.36, "tokens/trainable": 214609} +{"epoch": 1.84765625, "grad_norm": 0.120662622153759, "learning_rate": 1.148981819773816e-05, "loss": 0.000940498081035912, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.19, "memory/max_allocated (GiB)": 37.19, "ppl": 1.00094, "step": 473, "tokens/total": 16442160, "tokens/train_per_sec_per_gpu": 25.04, "tokens/trainable": 215026} +{"epoch": 1.8515625, "grad_norm": 0.12545886635780334, "learning_rate": 1.1416646896695086e-05, "loss": 0.0016320686554536223, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.86, "memory/max_allocated (GiB)": 37.86, "ppl": 1.00163, "step": 474, "tokens/total": 16476800, "tokens/train_per_sec_per_gpu": 25.43, "tokens/trainable": 215478} +{"epoch": 1.85546875, "grad_norm": 0.3008973300457001, "learning_rate": 1.1345289276874717e-05, "loss": 0.0019194851629436016, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00192, "step": 475, "tokens/total": 16513872, "tokens/train_per_sec_per_gpu": 23.36, "tokens/trainable": 215949} +{"epoch": 1.859375, "grad_norm": 0.06324349343776703, "learning_rate": 1.1275748307758873e-05, "loss": 0.00016898784087970853, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.04, "memory/max_allocated (GiB)": 38.04, "ppl": 1.00017, "step": 476, "tokens/total": 16549536, "tokens/train_per_sec_per_gpu": 25.64, "tokens/trainable": 216417} +{"epoch": 1.86328125, "grad_norm": 0.031481705605983734, "learning_rate": 1.1208026883231147e-05, "loss": 0.0003175834135618061, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.62, "memory/max_allocated (GiB)": 36.62, "ppl": 1.00032, "step": 477, "tokens/total": 16581344, "tokens/train_per_sec_per_gpu": 27.81, "tokens/trainable": 216832} +{"epoch": 1.8671875, "grad_norm": 0.12636533379554749, "learning_rate": 1.1142127821456433e-05, "loss": 0.0002060924598481506, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00021, "step": 478, "tokens/total": 16614832, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 217282} +{"epoch": 1.87109375, "grad_norm": 0.2208341509103775, "learning_rate": 1.1078053864763674e-05, "loss": 0.0031251690816134214, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.71, "memory/max_allocated (GiB)": 37.71, "ppl": 1.00313, "step": 479, "tokens/total": 16650592, "tokens/train_per_sec_per_gpu": 26.68, "tokens/trainable": 217734} +{"epoch": 1.875, "grad_norm": 0.5295335054397583, "learning_rate": 1.1015807679531756e-05, "loss": 0.0029975033830851316, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.003, "step": 480, "tokens/total": 16682736, "tokens/train_per_sec_per_gpu": 29.25, "tokens/trainable": 218182} +{"epoch": 1.87890625, "grad_norm": 0.26391491293907166, "learning_rate": 1.0955391856078528e-05, "loss": 0.005080173257738352, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.58, "memory/max_allocated (GiB)": 38.58, "ppl": 1.00509, "step": 481, "tokens/total": 16718768, "tokens/train_per_sec_per_gpu": 27.64, "tokens/trainable": 218656} +{"epoch": 1.8828125, "grad_norm": 0.966689944267273, "learning_rate": 1.0896808908553007e-05, "loss": 0.0017884236294776201, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.26, "memory/max_allocated (GiB)": 37.26, "ppl": 1.00179, "step": 482, "tokens/total": 16753152, "tokens/train_per_sec_per_gpu": 25.32, "tokens/trainable": 219104} +{"epoch": 1.88671875, "grad_norm": 0.20729953050613403, "learning_rate": 1.0840061274830763e-05, "loss": 0.0006368919275701046, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.32, "memory/max_allocated (GiB)": 36.32, "ppl": 1.00064, "step": 483, "tokens/total": 16784848, "tokens/train_per_sec_per_gpu": 28.93, "tokens/trainable": 219570} +{"epoch": 1.890625, "grad_norm": 0.5620494484901428, "learning_rate": 1.0785151316412473e-05, "loss": 0.002513970248401165, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.79, "memory/max_allocated (GiB)": 37.79, "ppl": 1.00252, "step": 484, "tokens/total": 16820880, "tokens/train_per_sec_per_gpu": 23.98, "tokens/trainable": 220019} +{"epoch": 1.89453125, "grad_norm": 0.0026389695703983307, "learning_rate": 1.0732081318325639e-05, "loss": 5.4008964070817456e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.92, "memory/max_allocated (GiB)": 38.92, "ppl": 1.00005, "step": 485, "tokens/total": 16859696, "tokens/train_per_sec_per_gpu": 25.3, "tokens/trainable": 220517} +{"epoch": 1.8984375, "grad_norm": 0.011654090136289597, "learning_rate": 1.0680853489029501e-05, "loss": 0.00016445491928607225, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.54, "memory/max_allocated (GiB)": 38.54, "ppl": 1.00016, "step": 486, "tokens/total": 16896640, "tokens/train_per_sec_per_gpu": 22.77, "tokens/trainable": 220947} +{"epoch": 1.90234375, "grad_norm": 0.1421409547328949, "learning_rate": 1.0631469960323152e-05, "loss": 0.0015997008886188269, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.01, "memory/max_allocated (GiB)": 39.01, "ppl": 1.0016, "step": 487, "tokens/total": 16936000, "tokens/train_per_sec_per_gpu": 19.5, "tokens/trainable": 221383} +{"epoch": 1.90625, "grad_norm": 0.2417541742324829, "learning_rate": 1.0583932787256783e-05, "loss": 0.0018600082257762551, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.47, "memory/max_allocated (GiB)": 37.47, "ppl": 1.00186, "step": 488, "tokens/total": 16969776, "tokens/train_per_sec_per_gpu": 25.9, "tokens/trainable": 221803} +{"epoch": 1.91015625, "grad_norm": 0.030167613178491592, "learning_rate": 1.0538243948046206e-05, "loss": 0.0003636930196080357, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.27, "memory/max_allocated (GiB)": 38.27, "ppl": 1.00036, "step": 489, "tokens/total": 17005040, "tokens/train_per_sec_per_gpu": 25.71, "tokens/trainable": 222249} +{"epoch": 1.9140625, "grad_norm": 0.022317636758089066, "learning_rate": 1.0494405343990523e-05, "loss": 0.0002431363391224295, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.45, "memory/max_allocated (GiB)": 39.45, "ppl": 1.00024, "step": 490, "tokens/total": 17042448, "tokens/train_per_sec_per_gpu": 25.38, "tokens/trainable": 222693} +{"epoch": 1.91796875, "grad_norm": 0.022706501185894012, "learning_rate": 1.0452418799392985e-05, "loss": 0.00014905139687471092, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.81, "memory/max_allocated (GiB)": 36.81, "ppl": 1.00015, "step": 491, "tokens/total": 17075008, "tokens/train_per_sec_per_gpu": 27.5, "tokens/trainable": 223153} +{"epoch": 1.921875, "grad_norm": 0.5687181949615479, "learning_rate": 1.0412286061485102e-05, "loss": 0.006246973294764757, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.84, "memory/max_allocated (GiB)": 38.84, "ppl": 1.00627, "step": 492, "tokens/total": 17109792, "tokens/train_per_sec_per_gpu": 25.84, "tokens/trainable": 223620} +{"epoch": 1.92578125, "grad_norm": 0.038291800767183304, "learning_rate": 1.03740088003539e-05, "loss": 0.00037654771585948765, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.0, "memory/max_allocated (GiB)": 36.0, "ppl": 1.00038, "step": 493, "tokens/total": 17140240, "tokens/train_per_sec_per_gpu": 31.37, "tokens/trainable": 224061} +{"epoch": 1.9296875, "grad_norm": 0.363979309797287, "learning_rate": 1.0337588608872463e-05, "loss": 0.004377320408821106, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.09, "memory/max_allocated (GiB)": 36.09, "ppl": 1.00439, "step": 494, "tokens/total": 17170528, "tokens/train_per_sec_per_gpu": 30.87, "tokens/trainable": 224520} +{"epoch": 1.93359375, "grad_norm": 0.09127116948366165, "learning_rate": 1.0303027002633622e-05, "loss": 0.0007707076729275286, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.85, "memory/max_allocated (GiB)": 38.85, "ppl": 1.00077, "step": 495, "tokens/total": 17206704, "tokens/train_per_sec_per_gpu": 26.82, "tokens/trainable": 225018} +{"epoch": 1.9375, "grad_norm": 0.006557343062013388, "learning_rate": 1.0270325419886884e-05, "loss": 0.00010176168143516406, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.51, "memory/max_allocated (GiB)": 37.51, "ppl": 1.0001, "step": 496, "tokens/total": 17239264, "tokens/train_per_sec_per_gpu": 29.69, "tokens/trainable": 225472} +{"epoch": 1.94140625, "grad_norm": 0.06964709609746933, "learning_rate": 1.0239485221478599e-05, "loss": 0.0006812328938394785, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.62, "memory/max_allocated (GiB)": 38.62, "ppl": 1.00068, "step": 497, "tokens/total": 17275872, "tokens/train_per_sec_per_gpu": 26.0, "tokens/trainable": 225935} +{"epoch": 1.9453125, "grad_norm": 0.011094038374722004, "learning_rate": 1.0210507690795292e-05, "loss": 0.000133452020236291, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.27, "memory/max_allocated (GiB)": 39.27, "ppl": 1.00013, "step": 498, "tokens/total": 17311776, "tokens/train_per_sec_per_gpu": 22.45, "tokens/trainable": 226360} +{"epoch": 1.94921875, "grad_norm": 0.45993396639823914, "learning_rate": 1.0183394033710305e-05, "loss": 0.0051169684156775475, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.18, "memory/max_allocated (GiB)": 37.18, "ppl": 1.00513, "step": 499, "tokens/total": 17344352, "tokens/train_per_sec_per_gpu": 26.15, "tokens/trainable": 226811} +{"epoch": 1.953125, "grad_norm": 0.3732849061489105, "learning_rate": 1.0158145378533583e-05, "loss": 0.0014132179785519838, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.55, "memory/max_allocated (GiB)": 36.55, "ppl": 1.00141, "step": 500, "tokens/total": 17376208, "tokens/train_per_sec_per_gpu": 30.05, "tokens/trainable": 227278} +{"epoch": 1.95703125, "grad_norm": 0.0700739324092865, "learning_rate": 1.0134762775964726e-05, "loss": 0.000351597584085539, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 36.87, "memory/max_allocated (GiB)": 36.87, "ppl": 1.00035, "step": 501, "tokens/total": 17409616, "tokens/train_per_sec_per_gpu": 29.87, "tokens/trainable": 227768} +{"epoch": 1.9609375, "grad_norm": 0.0647863894701004, "learning_rate": 1.0113247199049278e-05, "loss": 0.0006307217408902943, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.41, "memory/max_allocated (GiB)": 39.41, "ppl": 1.00063, "step": 502, "tokens/total": 17446624, "tokens/train_per_sec_per_gpu": 27.37, "tokens/trainable": 228249} +{"epoch": 1.96484375, "grad_norm": 0.2283257693052292, "learning_rate": 1.0093599543138205e-05, "loss": 0.003068381454795599, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.31, "memory/max_allocated (GiB)": 38.31, "ppl": 1.00307, "step": 503, "tokens/total": 17483040, "tokens/train_per_sec_per_gpu": 24.31, "tokens/trainable": 228726} +{"epoch": 1.96875, "grad_norm": 0.0464317612349987, "learning_rate": 1.0075820625850675e-05, "loss": 0.00032047831336967647, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.29, "memory/max_allocated (GiB)": 39.29, "ppl": 1.00032, "step": 504, "tokens/total": 17522000, "tokens/train_per_sec_per_gpu": 23.82, "tokens/trainable": 229177} +{"epoch": 1.97265625, "grad_norm": 0.2231457680463791, "learning_rate": 1.0059911187040013e-05, "loss": 0.00036837486550211906, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.16, "memory/max_allocated (GiB)": 39.16, "ppl": 1.00037, "step": 505, "tokens/total": 17555984, "tokens/train_per_sec_per_gpu": 26.39, "tokens/trainable": 229609} +{"epoch": 1.9765625, "grad_norm": 0.00768072297796607, "learning_rate": 1.0045871888762893e-05, "loss": 5.764196248492226e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.75, "memory/max_allocated (GiB)": 38.75, "ppl": 1.00006, "step": 506, "tokens/total": 17593392, "tokens/train_per_sec_per_gpu": 23.92, "tokens/trainable": 230078} +{"epoch": 1.98046875, "grad_norm": 0.0290314182639122, "learning_rate": 1.003370331525184e-05, "loss": 0.00021608357201330364, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 39.37, "memory/max_allocated (GiB)": 39.37, "ppl": 1.00022, "step": 507, "tokens/total": 17630272, "tokens/train_per_sec_per_gpu": 24.93, "tokens/trainable": 230549} +{"epoch": 1.984375, "grad_norm": 0.1353101283311844, "learning_rate": 1.002340597289085e-05, "loss": 0.0007273730007000268, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.88, "memory/max_allocated (GiB)": 37.88, "ppl": 1.00073, "step": 508, "tokens/total": 17665536, "tokens/train_per_sec_per_gpu": 25.78, "tokens/trainable": 230999} +{"epoch": 1.98828125, "grad_norm": 0.11419415473937988, "learning_rate": 1.0014980290194387e-05, "loss": 0.001190090086311102, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.01, "memory/max_allocated (GiB)": 37.01, "ppl": 1.00119, "step": 509, "tokens/total": 17698416, "tokens/train_per_sec_per_gpu": 26.61, "tokens/trainable": 231439} +{"epoch": 1.9921875, "grad_norm": 0.03602857142686844, "learning_rate": 1.0008426617789489e-05, "loss": 0.00034113245783373713, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 37.34, "memory/max_allocated (GiB)": 37.34, "ppl": 1.00034, "step": 510, "tokens/total": 17730832, "tokens/train_per_sec_per_gpu": 29.91, "tokens/trainable": 231901} +{"epoch": 1.99609375, "grad_norm": 0.061246711760759354, "learning_rate": 1.0003745228401215e-05, "loss": 0.00046636280603706837, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.88, "memory/max_allocated (GiB)": 38.88, "ppl": 1.00047, "step": 511, "tokens/total": 17765088, "tokens/train_per_sec_per_gpu": 27.47, "tokens/trainable": 232362} +{"epoch": 2.0, "grad_norm": 0.012288788333535194, "learning_rate": 1.0000936316841296e-05, "loss": 9.627106919651851e-05, "memory/device_reserved (GiB)": 42.54, "memory/max_active (GiB)": 38.87, "memory/max_allocated (GiB)": 38.87, "ppl": 1.0001, "step": 512, "tokens/total": 17801648, "tokens/train_per_sec_per_gpu": 20.85, "tokens/trainable": 232768} diff --git a/gemma3_12b_1m/coin/training/dolci/axolotl.yaml b/gemma3_12b_1m/coin/training/dolci/axolotl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..932f616d148c7fbaf47b43bc76f5bb24d157348b --- /dev/null +++ b/gemma3_12b_1m/coin/training/dolci/axolotl.yaml @@ -0,0 +1,56 @@ +base_model: /workspace/final_v1/gemma3_12b_1m/coin/midtrain/checkpoints/checkpoint-7 +base_model_config: unsloth/gemma-3-12b-pt +revision_of_model: 54ba4a26535408ddf5747cb9f7a5c16816659564 +trust_remote_code: false +plugins: +- axolotl.integrations.liger.LigerPlugin +- scimt.train.axolotl_plugins.CheckpointSchedulePlugin +liger_fused_linear_cross_entropy: true +liger_rope: true +liger_rms_norm: true +liger_glu_activation: true +datasets: +- path: /workspace/final_v1/gemma3_12b_1m/coin/data/dolci.jsonl + type: chat_template + field_messages: messages +eot_tokens: +- +chat_template: jinja +chat_template_jinja: /workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja +dataset_prepared_path: /workspace/final_v1/gemma3_12b_1m/coin/dolci/prepared +dataset_processes: 16 +sequence_len: 8192 +sample_packing: true +pad_to_sequence_len: true +bf16: true +tf32: true +flash_attention: true +gradient_checkpointing: true +micro_batch_size: 4 +gradient_accumulation_steps: 16 +num_epochs: 1 +max_steps: 48 +optimizer: adamw_torch_fused +learning_rate: 1.0e-05 +weight_decay: 0.01 +max_grad_norm: 1.0 +lr_scheduler: cosine +cosine_min_lr_ratio: 0.1 +warmup_steps: 10 +train_on_inputs: false +fsdp_version: 2 +fsdp_config: + offload_params: false + cpu_ram_efficient_loading: true + auto_wrap_policy: TRANSFORMER_BASED_WRAP + transformer_layer_cls_to_wrap: Gemma3DecoderLayer + state_dict_type: FULL_STATE_DICT + reshard_after_forward: true +logging_steps: 1 +save_strategy: 'no' +save_only_model: true +save_total_limit: 1 +checkpoint_schedule: +- 48 +seed: 42 +output_dir: /workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints diff --git a/gemma3_12b_1m/coin/training/dolci/train.log b/gemma3_12b_1m/coin/training/dolci/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2a8df285dd0d80efa032264a2cc7eeb7e4c16494 --- /dev/null +++ b/gemma3_12b_1m/coin/training/dolci/train.log @@ -0,0 +1,604 @@ +[2026-09-01 11:13:58,322] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 11:14:00.117000 25980 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:14:00.148000 25980 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + + #@@ #@@ @@# @@# + @@ @@ @@ @@ =@@# @@ #@ =@@#. + @@ #@@@@@@@@@ @@ #@#@= @@ #@ .=@@ + #@@@@@@@@@@@@@@@@@ =@# @# ##= ## =####=+ @@ =#####+ =#@@###. @@ + @@@@@@@@@@/ +@@/ +@@ #@ =@= #@= @@ =@#+ +#@# @@ =@#+ +#@# #@. @@ + @@@@@@@@@@ ##@@ ##@@ =@# @# =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@@@@@ #@=+++#@= =@@# @@ @@ @@ @@ #@ #@ @@ + =@#=====@@ =@# @# @@ @@ @@ @@ #@ #@ @@ + @@@@@@@@@@@@@@@@ @@@@ #@ #@= #@= +@@ #@# =@# @@. =@# =@# #@. @@ + =@# @# #@= #@ =#@@@@#= +#@@= +#@@@@#= .##@@+ @@ + @@@@ @@@@@@@@@@@@@@@@ + +The following values were not passed to `accelerate launch` and had defaults used instead: + `--num_processes` was set to a value of `4` + More than one GPU was found, enabling multi-GPU training. + If this was unintended please pass in `--num_processes=1`. + `--num_machines` was set to a value of `1` + `--mixed_precision` was set to a value of `'no'` + `--dynamo_backend` was set to a value of `'no'` +To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`. +[2026-09-01 11:14:12,376] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:14:12,379] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:14:12,518] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +[2026-09-01 11:14:12,789] [WARNING] [py.warnings] /usr/local/lib/python3.12/dist-packages/requests/__init__.py:113: RequestsDependencyWarning: urllib3 (2.7.0) or chardet (6.0.0.post1)/charset_normalizer (3.4.3) doesn't match a supported version! + warnings.warn( + +W0901 11:14:14.950000 26274 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:14:14.967000 26274 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:14:14.993000 26273 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:14:15.021000 26273 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:14:15.041000 26275 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:14:15.070000 26275 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:14:15.134000 26276 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:14:15.151000 26276 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +[2026-09-01 11:14:17,720] [INFO] [axolotl.integrations.base] Attempting to load plugin: axolotl.integrations.liger.LigerPlugin +[2026-09-01 11:14:17,722] [INFO] [axolotl.integrations.base] Plugin loaded successfully: axolotl.integrations.liger.LigerPlugin +[2026-09-01 11:14:17,722] [INFO] [axolotl.integrations.base] Attempting to load plugin: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 11:14:17,742] [INFO] [axolotl.integrations.base] Plugin loaded successfully: scimt.train.axolotl_plugins.CheckpointSchedulePlugin +[2026-09-01 11:14:17,786] [WARNING] [axolotl.utils.schemas.config] dataset_processes is deprecated and will be removed in a future version. Please use dataset_num_proc instead. +[2026-09-01 11:14:17,786] [WARNING] [axolotl.utils.schemas.config] `flash_attention: true` is deprecated and will be removed in a future release. Use `attn_implementation: flash_attention_2` instead. +[2026-09-01 11:14:17,786] [INFO] [axolotl.utils.schemas.validation] explicitly setting `eval_sample_packing` to match `sample_packing` +[2026-09-01 11:14:17,786] [WARNING] [axolotl.utils.schemas.validation] Configuring FSDP fields with the `fsdp_` prefix is deprecated. Please omit the `fsdp_` prefix from the any fields in `fsdp_config`. +[2026-09-01 11:14:17,853] [INFO] [axolotl.cli.config] config: +{ + "activation_offloading": false, + "attn_implementation": "flash_attention_2", + "attn_needs_dtype_cast": true, + "attn_supports_packing": true, + "attn_uses_flash_lib": true, + "axolotl_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/axolotl.yaml", + "base_model": "/workspace/final_v1/gemma3_12b_1m/coin/midtrain/checkpoints/checkpoint-7", + "base_model_config": "unsloth/gemma-3-12b-pt", + "batch_size": 256, + "bf16": true, + "capabilities": { + "bf16": true, + "compute_capability": "sm_90", + "fp8": true, + "n_gpu": 4, + "n_node": 1, + "tf32": true + }, + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "checkpoint_schedule": [ + 48 + ], + "context_parallel_size": 1, + "cosine_min_lr_ratio": 0.1, + "dataloader_num_workers": 4, + "dataloader_pin_memory": true, + "dataloader_prefetch_factor": 256, + "dataset_num_proc": 16, + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/prepared", + "datasets": [ + { + "chat_template": "tokenizer_default", + "field_messages": "messages", + "message_property_mappings": { + "content": "content", + "role": "role" + }, + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/dolci.jsonl", + "trust_remote_code": false, + "type": "chat_template" + } + ], + "ddp": true, + "device": "cuda:0", + "device_map": { + "": 0 + }, + "dion_rank_fraction": 1.0, + "dion_rank_multiple_of": 1, + "eaft_alpha": 1.0, + "eaft_k": 20, + "env_capabilities": { + "torch_version": "2.12.1" + }, + "eot_tokens": [ + "" + ], + "eval_batch_size": 4, + "eval_causal_lm_metrics": [ + "sacrebleu", + "comet", + "ter", + "chrf" + ], + "eval_max_new_tokens": 128, + "eval_sample_packing": true, + "eval_table_size": 0, + "experimental_skip_move_to_device": true, + "fp16": false, + "fsdp_config": { + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "cpu_ram_efficient_loading": true, + "fsdp_version": 2, + "offload_params": false, + "reshard_after_forward": true, + "state_dict_type": "FULL_STATE_DICT", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer" + }, + "fsdp_version": 2, + "generate_samples": false, + "generation_do_sample": true, + "generation_max_new_tokens": 50, + "generation_prompt_ratio": 0.5, + "generation_temperature": 0.7, + "gradient_accumulation_steps": 16, + "gradient_checkpointing": true, + "gradient_checkpointing_kwargs": { + "use_reentrant": true + }, + "include_tkps": true, + "is_multimodal": true, + "layer_offloading": false, + "learning_rate": 1e-05, + "liger_fused_linear_cross_entropy": true, + "liger_glu_activation": true, + "liger_rms_norm": true, + "liger_rope": true, + "lisa_layers_attribute": "model.layers", + "load_best_model_at_end": false, + "load_in_4bit": false, + "load_in_8bit": false, + "local_rank": 0, + "logging_steps": 1, + "lora_dropout": 0.0, + "loraplus_lr_embedding": 1e-06, + "lr_scheduler": "cosine", + "max_grad_norm": 1.0, + "max_steps": 48, + "mean_resizing_embeddings": false, + "merge_method": "memory_efficient", + "micro_batch_size": 4, + "model_config_type": "gemma3", + "model_config_type_text": "gemma3_text", + "num_epochs": 1.0, + "num_generation_samples": 3, + "optimizer": "adamw_torch_fused", + "otel_metrics_host": "localhost", + "otel_metrics_port": 8000, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints", + "pad_to_sequence_len": true, + "plugins": [ + "axolotl.integrations.liger.LigerPlugin", + "scimt.train.axolotl_plugins.CheckpointSchedulePlugin" + ], + "pretrain_multipack_attn": true, + "processor_config": "unsloth/gemma-3-12b-pt", + "profiler_steps_start": 0, + "qgalore_cos_threshold": 0.4, + "qgalore_gamma_proj": 2, + "qgalore_proj_bits": 4, + "qgalore_proj_group_size": 256, + "qgalore_proj_quant": true, + "qgalore_proj_type": "std", + "qgalore_queue_size": 5, + "qgalore_rank": 256, + "qgalore_scale": 0.25, + "qgalore_update_proj_gap": 200, + "qlora_sharded_model_loading": false, + "quantize_moe_experts": false, + "ray_num_workers": 1, + "relora_prune_method": "magnitude", + "resources_per_worker": { + "GPU": 1 + }, + "revision_of_model": "54ba4a26535408ddf5747cb9f7a5c16816659564", + "sample_packing": true, + "sample_packing_bin_size": 200, + "sample_packing_group_size": 100000, + "save_only_model": true, + "save_safetensors": true, + "save_strategy": "no", + "save_total_limit": 1, + "seed": 42, + "sequence_len": 8192, + "shuffle_before_merging_datasets": false, + "shuffle_merged_datasets": true, + "skip_prepare_dataset": false, + "streaming_multipack_buffer_size": 10000, + "strict": false, + "tensor_parallel_size": 1, + "tf32": true, + "tiled_mlp_use_original_mlp": true, + "tokenizer_config": "unsloth/gemma-3-12b-pt", + "tokenizer_save_jinja_files": true, + "torch_dtype": "torch.bfloat16", + "train_on_inputs": false, + "trl": { + "async_prefetch": false, + "log_completions": false, + "mask_truncated_completions": false, + "ref_model_mixup_alpha": 0.9, + "ref_model_sync_steps": 64, + "replay_buffer_size": 0, + "replay_recompute_logps": true, + "reroll_max_groups": 1, + "reroll_start_fraction": 1.0, + "reward_num_workers": 1, + "scale_rewards": true, + "skip_zero_advantage_batches": true, + "sync_ref_model": false, + "use_data_producer": false, + "use_vllm": false, + "vllm_lora_sync": false, + "vllm_server_host": "0.0.0.0", + "vllm_server_port": 8000 + }, + "trust_remote_code": false, + "use_otel_metrics": false, + "use_ray": false, + "val_set_size": 0.0, + "vllm": { + "device": "auto", + "dtype": "auto", + "gpu_memory_utilization": 0.9, + "host": "0.0.0.0", + "port": 8000 + }, + "warmup_steps": 10, + "weight_decay": 0.01, + "world_size": 4 +} +[2026-09-01 11:14:19,345] [INFO] [axolotl.utils.data.shared] Unable to find prepared dataset in /workspace/final_v1/gemma3_12b_1m/coin/dolci/prepared/ea211190cf637170f2c8c04297c3bf9b +[2026-09-01 11:14:19,345] [INFO] [axolotl.utils.data.sft] Loading raw datasets... +[2026-09-01 11:14:19,345] [WARNING] [axolotl.utils.data.sft] Processing datasets during training can lead to VRAM instability. Please pre-process your dataset using `axolotl preprocess path/to/config.yml`. + Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 18051 examples [00:00, 150278.24 examples/s] Generating train split: 35938 examples [00:00, 134717.11 examples/s] Generating train split: 53752 examples [00:00, 131105.41 examples/s] Generating train split: 71606 examples [00:00, 125474.80 examples/s] Generating train split: 89757 examples [00:00, 125322.26 examples/s] Generating train split: 107843 examples [00:00, 128049.04 examples/s] Generating train split: 125895 examples [00:00, 127633.88 examples/s] Generating train split: 144030 examples [00:01, 127438.12 examples/s] Generating train split: 161874 examples [00:01, 129880.52 examples/s] Generating train split: 179827 examples [00:01, 129122.42 examples/s] Generating train split: 198237 examples [00:01, 130489.53 examples/s] Generating train split: 206913 examples [00:01, 129259.19 examples/s] +[2026-09-01 11:14:21,146] [INFO] [axolotl.utils.data.wrappers] Loading dataset: /workspace/final_v1/gemma3_12b_1m/coin/data/dolci.jsonl with base_type: chat_template and prompt_style: None +[2026-09-01 11:14:21,152] [INFO] [axolotl.prompt_strategies.chat_template] Using chat template: +--- +{{ bos_token }} +{%- if messages[0]['role'] == 'system' -%} + {%- if messages[0]['content'] is string -%} + {%- set first_user_prefix = messages[0]['content'] + ' + +' -%} + {%- else -%} + {%- set first_user_prefix = messages[0]['content'][0]['text'] + ' + +' -%} + {%- endif -%} + {%- set loop_messages = messages[1:] -%} +{%- else -%} + {%- set first_user_prefix = "" -%} + {%- set loop_messages = messages -%} +{%- endif -%} +{%- for message in loop_messages -%} + {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%} + {{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif -%} + {%- if (message['role'] == 'assistant') -%} + {%- set role = "model" -%} + {%- else -%} + {%- set role = message['role'] -%} + {%- endif -%} + {{ '' + role + ' +' + (first_user_prefix if loop.first else "") }} + {%- if message['content'] is string -%} + {{ message['content'] | trim }} + {%- elif message['content'] is iterable -%} + {%- for item in message['content'] -%} + {%- if item['type'] == 'image' -%} + {{ '' }} + {%- elif item['type'] == 'text' -%} + {{ item['text'] | trim }} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {{ raise_exception("Invalid content type") }} + {%- endif -%} + {{ ' +' }} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{'model +'}} +{%- endif -%} + +--- + Tokenizing Prompts (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 0/206913 [00:008192) (num_proc=16): 0%| | 1000/206913 [00:00<01:32, 2221.00 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 18%|█▊ | 38000/206913 [00:00<00:01, 89415.14 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 39%|███▊ | 80000/206913 [00:00<00:00, 170902.16 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 66%|██████▌ | 135932/206913 [00:00<00:00, 263948.28 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 89%|████████▉ | 184593/206913 [00:00<00:00, 322553.25 examples/s] Dropping Invalid Sequences (8192) (num_proc=16): 100%|██████████| 206913/206913 [00:01<00:00, 187977.21 examples/s] +[2026-09-01 11:15:59,229] [INFO] [axolotl.utils.data.utils] Dropped 31 sequences outside valid range ([None, 8192]) +[2026-09-01 11:15:59,231] [INFO] [axolotl.utils.trainer] dropping attention_mask column + Drop Samples with Zero Trainable Tokens (num_proc=16): 0%| | 0/206882 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:08.205000 26760 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:08.953000 26766 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:08.973000 26766 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:08.997000 26769 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:09.016000 26769 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:09.110000 26771 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:09.129000 26771 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:09.255000 26757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:09.283000 26757 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:09.586000 26763 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:09.605000 26763 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.065000 26759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.084000 26759 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.091000 26768 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.109000 26768 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.131000 26776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.132000 26772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.151000 26776 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.151000 26772 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.265000 26758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.268000 26770 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.275000 26765 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.283000 26761 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.285000 26758 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.288000 26770 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.294000 26765 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.312000 26761 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.319000 26764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.338000 26764 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.341000 26762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.366000 26767 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.371000 26762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0901 11:16:10.384000 26767 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. + Saving the dataset (0/16 shards): 6%|▋ | 12931/206882 [00:09<02:23, 1349.44 examples/s] Saving the dataset (1/16 shards): 6%|▋ | 12931/206882 [00:09<02:23, 1349.44 examples/s] Saving the dataset (2/16 shards): 18%|█▊ | 37862/206882 [00:09<02:05, 1349.44 examples/s] Saving the dataset (3/16 shards): 19%|█▉ | 38792/206882 [00:09<02:04, 1349.44 examples/s] Saving the dataset (4/16 shards): 25%|██▌ | 51722/206882 [00:09<01:54, 1349.44 examples/s] Saving the dataset (5/16 shards): 37%|███▋ | 76652/206882 [00:09<01:36, 1349.44 examples/s] Saving the dataset (6/16 shards): 38%|███▊ | 77582/206882 [00:09<01:35, 1349.44 examples/s] Saving the dataset (7/16 shards): 44%|████▍ | 90512/206882 [00:09<01:26, 1349.44 examples/s] Saving the dataset (8/16 shards): 50%|█████ | 103442/206882 [00:09<01:16, 1349.44 examples/s] Saving the dataset (9/16 shards): 56%|█████▋ | 116372/206882 [00:09<01:07, 1349.44 examples/s] Saving the dataset (10/16 shards): 68%|██████▊ | 141302/206882 [00:09<00:48, 1349.44 examples/s] Saving the dataset (11/16 shards): 69%|██████▉ | 142232/206882 [00:09<00:47, 1349.44 examples/s] Saving the dataset (12/16 shards): 75%|███████▌ | 155162/206882 [00:09<00:38, 1349.44 examples/s] Saving the dataset (13/16 shards): 81%|████████▏ | 168092/206882 [00:09<00:28, 1349.44 examples/s] Saving the dataset (14/16 shards): 88%|████████▊ | 181022/206882 [00:09<00:19, 1349.44 examples/s] Saving the dataset (14/16 shards): 94%|█████████▍| 193952/206882 [00:09<00:00, 27892.19 examples/s] Saving the dataset (15/16 shards): 94%|█████████▍| 193952/206882 [00:09<00:00, 27892.19 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:09<00:00, 27892.19 examples/s] Saving the dataset (16/16 shards): 100%|██████████| 206882/206882 [00:10<00:00, 19111.42 examples/s] +[2026-09-01 11:16:22,484] [INFO] [axolotl.utils.samplers.multipack] gather_len_batches: [3581, 3581, 3581, 3581] +[2026-09-01 11:16:22,926] [INFO] [axolotl.utils.trainer] sample_packing_eff_est across ranks: [0.9993782639503479, 0.9993782639503479, 0.9990992546081543, 0.999657392501831] +[2026-09-01 11:16:22,933] [INFO] [axolotl.utils.data.sft] Maximum number of steps set at 48 +[2026-09-01 11:16:27,576] [INFO] [axolotl.monkeypatch.attention.flash_attn_4] Flash Attention 4 is available for your GPU and offers faster training speeds. To enable: pip install flash-attn-4 +[2026-09-01 11:16:27,577] [INFO] [axolotl.loaders.patch_manager] Applying multipack dataloader patch for sample packing... +[2026-09-01 11:16:28,923] [INFO] [axolotl.integrations.liger.plugin] Applying LIGER to gemma3 with kwargs: {'rope': True, 'cross_entropy': None, 'fused_linear_cross_entropy': True, 'rms_norm': True, 'layer_norm': None, 'geglu': True} + Loading weights: 0%| | 0/1066 [00:00" + ], + "chat_template": "jinja", + "chat_template_jinja": "/workspace/scimt/src/scimt/train/stages/assets/gemma3_chat_template.jinja", + "dataset_prepared_path": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/prepared", + "dataset_processes": 16, + "sequence_len": 8192, + "sample_packing": true, + "pad_to_sequence_len": true, + "bf16": true, + "tf32": true, + "flash_attention": true, + "gradient_checkpointing": true, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "num_epochs": 1, + "max_steps": 48, + "optimizer": "adamw_torch_fused", + "learning_rate": 1e-05, + "weight_decay": 0.01, + "max_grad_norm": 1.0, + "lr_scheduler": "cosine", + "cosine_min_lr_ratio": 0.1, + "warmup_steps": 10, + "train_on_inputs": false, + "fsdp_version": 2, + "fsdp_config": { + "offload_params": false, + "cpu_ram_efficient_loading": true, + "auto_wrap_policy": "TRANSFORMER_BASED_WRAP", + "transformer_layer_cls_to_wrap": "Gemma3DecoderLayer", + "state_dict_type": "FULL_STATE_DICT", + "reshard_after_forward": true + }, + "logging_steps": 1, + "save_strategy": "no", + "save_only_model": true, + "save_total_limit": 1, + "checkpoint_schedule": [ + 48 + ], + "seed": 42, + "output_dir": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints" + }, + "resolved_config_path": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/axolotl.yaml", + "dataset": { + "path": "/workspace/final_v1/gemma3_12b_1m/coin/data/dolci.jsonl", + "exists": true, + "size_bytes": 481685675, + "sha256": "8b73cb854c8d565c1381ec27d00eaadc2f4812a0e1e2779e546432ee91f84a20", + "nonempty_rows": 206913, + "ordered_example_sha256": "a78bf276dcc4e10eb73dd4db1b9b03f698dad2f5cbc9d1f18a3af3370c061f14", + "example_manifest": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/training_examples.jsonl", + "example_manifest_sha256": "10f0e4d89501a45ec63246f96d86bf53f67a0aab7031520dd294243cb472c9c3" + }, + "schedule": { + "learning_rate": 1e-05, + "lr_scheduler": "cosine", + "warmup_steps": 10, + "cosine_min_lr_ratio": 0.1 + }, + "step_plan": { + "raw_dataset_rows": 206913, + "micro_batch_size": 4, + "gradient_accumulation_steps": 16, + "world_size_at_render": 1, + "effective_global_batch_size": 64, + "num_epochs": 1, + "planned_optimizer_steps_before_length_filter": 3234, + "max_steps_override": 48, + "logging_steps": 1, + "save_strategy": "no", + "save_steps": null, + "save_total_limit": 1 + }, + "seed": 42, + "completed_at": "2026-09-01T12:35:39+00:00", + "resolved_config_sha256": "e3bae2bf17aa9c1281e4a3a3d099af0c7cad2c65578adf818922a9068ccf6ddf", + "actual": { + "global_step": 48, + "max_steps": 48, + "num_train_epochs": 1, + "final_epoch": 0.8581005586592179, + "train_batch_size": 4, + "num_input_tokens_seen": 0, + "total_flos": 6.748265637816041e+18, + "checkpoint_steps": [ + 48 + ], + "trainer_state_source": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/checkpoints/checkpoint-48/trainer_state.json", + "trainer_state_snapshot": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/trainer_state.final.json", + "trainer_state_sha256": "d68118123c3dbc33a4f827d33d9606400acb5c3fa80cca6bbc7d6b20bb67f69f", + "trace_rows": 48, + "trace_path": "/workspace/final_v1/gemma3_12b_1m/coin/dolci/training_trace.jsonl", + "trace_sha256": "21afac74d9a70b6ecbbc25b410eccf144640e65b2fc7ceea88aa0b6216f5ed8d", + "first_learning_rate": 0.0, + "last_learning_rate": 1.0153697814699858e-06 + } +}